diff options
| author | Thibault Saunier <tsaunier@igalia.com> | 2022-08-16 17:36:53 +0300 |
|---|---|---|
| committer | Thibault Saunier <tsaunier@igalia.com> | 2022-10-18 16:18:53 +0300 |
| commit | 5e7537953c28ca2d2586f6078c24cccf1c99a27d (patch) | |
| tree | aa155f05a0d76f921272b8f331dc24ffb0f0df6b /net | |
| parent | 020c7e2900ff6b1db9a68c7abae8d282c9e896fb (diff) | |
webrtc: Move to net/webrtc
Diffstat (limited to 'net')
51 files changed, 16369 insertions, 0 deletions
diff --git a/net/webrtc/Cargo.lock b/net/webrtc/Cargo.lock new file mode 100644 index 00000000..a7f6c23f --- /dev/null +++ b/net/webrtc/Cargo.lock @@ -0,0 +1,2071 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" +dependencies = [ + "autocfg", + "concurrent-queue", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "winapi", +] + +[[package]] +name = "async-lock" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-native-tls" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d57d4cec3c647232e1094dc013546c0b33ce785d8aeb251e1f20dfaf8a9a13fe" +dependencies = [ + "futures-util", + "native-tls", + "thiserror", + "url", +] + +[[package]] +name = "async-process" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02111fd8655a613c25069ea89fc8d9bb89331fa77486eb3bc059ee757cfa481c" +dependencies = [ + "async-io", + "autocfg", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "libc", + "once_cell", + "signal-hook", + "winapi", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-attributes", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" + +[[package]] +name = "async-tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b71b31561643aa8e7df3effe284fa83ab1a840e52294c5f4bd7bfd8b2becbb" +dependencies = [ + "async-native-tls", + "async-std", + "futures-io", + "futures-util", + "log", + "pin-project-lite", + "tungstenite", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-expr" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aacacf4d96c24b2ad6eb8ee6df040e4f27b0d0b39a5710c30091baa830485db" +dependencies = [ + "smallvec", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "clap" +version = "4.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef582e2c00a63a0c0aa1fb4a4870781c4f5729f51196d3537fa7c1c1992eaa3" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f169caba89a7d512b5418b09864543eeb4d497416c917d7137863bd2076ad" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "cxx" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f83d0ebf42c6eafb8d7c52f7e5f2d3003b89c7aa4fd2b79229209459a849af8" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d050484b55975889284352b0ffc2ecbda25c0c55978017c132b29ba0818a86" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d2199b00553eda8012dfec8d3b1c75fce747cf27c169a270b3b99e3448ab78" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb67a6de1f602736dd7eaead0080cf3435df806c61b24b13328db128c58868f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-executor" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gio-sys" +version = "0.16.0" +source = "git+https://github.com/gtk-rs/gtk-rs-core#bc63686118d9382a5e46260ed040e2028e88321c" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", + "winapi", +] + +[[package]] +name = "glib" +version = "0.16.0" +source = "git+https://github.com/gtk-rs/gtk-rs-core#bc63686118d9382a5e46260ed040e2028e88321c" +dependencies = [ + "bitflags", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys", + "glib-macros", + "glib-sys", + "gobject-sys", + "libc", + "once_cell", + "smallvec", + "thiserror", +] + +[[package]] +name = "glib-macros" +version = "0.16.0" +source = "git+https://github.com/gtk-rs/gtk-rs-core#bc63686118d9382a5e46260ed040e2028e88321c" +dependencies = [ + "anyhow", + "heck", + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "glib-sys" +version = "0.16.0" +source = "git+https://github.com/gtk-rs/gtk-rs-core#bc63686118d9382a5e46260ed040e2028e88321c" +dependencies = [ + "libc", + "system-deps", +] + +[[package]] +name = "gloo-timers" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "gobject-sys" +version = "0.16.0" +source = "git+https://github.com/gtk-rs/gtk-rs-core#bc63686118d9382a5e46260ed040e2028e88321c" +dependencies = [ + "glib-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gst-plugin-version-helper" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6a4dd1cb931cc6b49af354a68f21b3aee46b5b07370215d942f3a71542123f" +dependencies = [ + "chrono", +] + +[[package]] +name = "gstreamer" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "bitflags", + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib", + "gstreamer-sys", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations", + "paste", + "pretty-hex", + "serde", + "serde_bytes", + "thiserror", +] + +[[package]] +name = "gstreamer-app" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "bitflags", + "futures-core", + "futures-sink", + "glib", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-base" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "atomic_refcell", + "bitflags", + "cfg-if", + "glib", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-rtp" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "bitflags", + "glib", + "gstreamer", + "gstreamer-rtp-sys", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-rtp-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sdp" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-sdp-sys", +] + +[[package]] +name = "gstreamer-sdp-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-utils" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "gstreamer", + "gstreamer-app", + "gstreamer-video", + "once_cell", + "thiserror", +] + +[[package]] +name = "gstreamer-video" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "bitflags", + "cfg-if", + "futures-channel", + "glib", + "gstreamer", + "gstreamer-base", + "gstreamer-video-sys", + "libc", + "once_cell", + "serde", +] + +[[package]] +name = "gstreamer-video-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-webrtc" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-sdp", + "gstreamer-webrtc-sys", + "libc", +] + +[[package]] +name = "gstreamer-webrtc-sys" +version = "0.19.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs#7721030c15c9692bae262e2d5a4e696a743ec96e" +dependencies = [ + "glib-sys", + "gstreamer-sdp-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "human_bytes" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a0d4dc39ec942e44c1c306aa196da67f2bd6a30dc7b4a475465c13ccf28817" + +[[package]] +name = "iana-time-zone" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.135" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" + +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", + "value-bag", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "muldiv" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5136edda114182728ccdedb9f5eda882781f35fa6e80cc360af12a8932507f3" + +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" + +[[package]] +name = "openssl" +version = "0.10.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-operations" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0" +dependencies = [ + "paste", +] + +[[package]] +name = "os_str_bytes" +version = "6.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "paste" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "polling" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "pretty-hex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" + +[[package]] +name = "proc-macro-crate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +dependencies = [ + "once_cell", + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + +[[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc50e8183eeeb6178dcb167ae34a8051d63535023ae38b5d8d12beae193d37b" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "system-deps" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a45a1c4c9015217e12347f2a411b57ce2c4fc543913b14b6fe40483328e709" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "test-log" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f0c854faeb68a048f0f2dc410c5ddae3bf83854ef0e4977d58306a5edef50e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tungstenite" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "native-tls", + "rand", + "sha-1", + "thiserror", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "uuid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83" +dependencies = [ + "getrandom", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "value-bag" +version = "1.0.0-alpha.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +dependencies = [ + "ctor", + "version_check", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe88247b92c1df6b6de80ddc290f3976dbdf2f5f5d3fd049a9fb598c6dd5ca73" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webrtcsink" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-native-tls", + "async-std", + "async-tungstenite", + "chrono", + "clap", + "fastrand", + "futures", + "gst-plugin-version-helper", + "gstreamer", + "gstreamer-app", + "gstreamer-rtp", + "gstreamer-sdp", + "gstreamer-utils", + "gstreamer-video", + "gstreamer-webrtc", + "human_bytes", + "once_cell", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tracing", + "tracing-log", + "tracing-subscriber", + "uuid", + "webrtcsink-protocol", +] + +[[package]] +name = "webrtcsink-protocol" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "webrtcsink-signalling" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-native-tls", + "async-std", + "async-tungstenite", + "clap", + "futures", + "pin-project-lite", + "serde", + "serde_json", + "test-log", + "thiserror", + "tracing", + "tracing-log", + "tracing-subscriber", + "uuid", + "webrtcsink-protocol", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/net/webrtc/Cargo.toml b/net/webrtc/Cargo.toml new file mode 100644 index 00000000..b8454061 --- /dev/null +++ b/net/webrtc/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] + +members = [ + "plugins", + "protocol", + "signalling", +] diff --git a/net/webrtc/LICENSE b/net/webrtc/LICENSE new file mode 100644 index 00000000..14e2f777 --- /dev/null +++ b/net/webrtc/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/net/webrtc/README.md b/net/webrtc/README.md new file mode 100644 index 00000000..5380f260 --- /dev/null +++ b/net/webrtc/README.md @@ -0,0 +1,224 @@ +# webrtcsink + +All-batteries included GStreamer WebRTC producer, that tries its best to do The Right Thing™. + +## Use case + +The [webrtcbin] element in GStreamer is extremely flexible and powerful, but using +it can be a difficult exercise. When all you want to do is serve a fixed set of streams +to any number of consumers, `webrtcsink` (which wraps `webrtcbin` internally) can be a +useful alternative. + +[webrtcbin]: https://gstreamer.freedesktop.org/documentation/webrtc/index.html + +## Features + +`webrtcsink` implements the following features: + +* Built-in signaller: when using the default signalling server, this element will + perform signalling without requiring application interaction. + This makes it usable directly from `gst-launch`. + +* Application-provided signalling: `webrtcsink` can be instantiated by an application + with a custom signaller. That signaller must be a GObject, and must implement the + `Signallable` interface as defined [here](plugins/src/webrtcsink/mod.rs). The + [default signaller](plugins/src/signaller/mod.rs) can be used as an example. + + An [example project] is also available to use as a boilerplate for implementing + and using a custom signaller. + +* Sandboxed consumers: when a consumer is added, its encoder / payloader / webrtcbin + elements run in a separately managed pipeline. This provides a certain level of + sandboxing, as opposed to having those elements running inside the element itself. + + It is important to note that at this moment, encoding is not shared between consumers. + While this is not on the roadmap at the moment, nothing in the design prevents + implementing this optimization. + +* Congestion control: the element leverages transport-wide congestion control + feedback messages in order to adapt the bitrate of individual consumers' video + encoders to the available bandwidth. + +* Configuration: the level of user control over the element is slowly expanding, + consult `gst-inspect-1.0` for more information on the available properties and + signals. + +* Packet loss mitigation: webrtcsink now supports sending protection packets for + Forward Error Correction, modulating the amount as a function of the available + bandwidth, and can honor retransmission requests. Both features can be disabled + via properties. + +It is important to note that full control over the individual elements used by +`webrtcsink` is *not* on the roadmap, as it will act as a black box in that respect, +for example `webrtcsink` wants to reserve control over the bitrate for congestion +control. + +A signal is now available however for the application to provide the initial +configuration for the encoders `webrtcsink` instantiates. + +If more granular control is required, applications should use `webrtcbin` directly, +`webrtcsink` will focus on trying to just do the right thing, although it might +expose more interfaces to guide and tune the heuristics it employs. + +[example project]: https://github.com/centricular/webrtcsink-custom-signaller + +## Building + +### Prerequisites + +The element has only been tested for now against GStreamer main. + +For testing, it is recommended to simply build GStreamer locally and run +in the uninstalled devenv. + +> Make sure to install the development packages for some codec libraries +> beforehand, such as libx264, libvpx and libopusenc, exact names depend +> on your distribution. + +``` +git clone --depth 1 --single-branch --branch main https://gitlab.freedesktop.org/gstreamer/gstreamer +cd gstreamer +meson build +ninja -C build +ninja -C build devenv +``` + +### Compiling + +``` shell +cargo build +``` + +## Usage + +Open three terminals. In the first, run: + +``` shell +WEBRTCSINK_SIGNALLING_SERVER_LOG=debug cargo run --bin server +``` + +In the second, run: + +``` shell +python3 -m http.server -d www/ +``` + +In the third, run: + +``` shell +export GST_PLUGIN_PATH=$PWD/target/debug:$GST_PLUGIN_PATH +gst-launch-1.0 webrtcsink name=ws videotestsrc ! ws. audiotestsrc ! ws. +``` + +When the pipeline above is running succesfully, open a browser and +point it to the http server: + +``` shell +xdg-open http://127.0.0.1:8000 +``` + +You should see an identifier listed in the left-hand panel, click on +it. You should see a test video stream, and hear a test tone. + +## Configuration + +The element itself can be configured through its properties, see +`gst-inspect-1.0 webrtcsink` for more information about that, in addition the +default signaller also exposes properties for configuring it, in +particular setting the signalling server address, those properties +can be accessed through the `gst::ChildProxy` interface, for example +with gst-launch: + +``` shell +gst-launch-1.0 webrtcsink signaller::address="ws://127.0.0.1:8443" .. +``` + +The signaller object can not be inspected, refer to [the source code] +for the list of properties. + +[the source code]: plugins/src/signaller/imp.rs + + +### Enable 'navigation' a.k.a user interactivity with the content + +`webrtcsink` implements the [`GstNavigation`] interface which allows interacting +with the content, for example move with your mouse, entering keys with the +keyboard, etc... On top of that a `WebRTCDataChannel` based protocol has been +implemented and can be activated with the `enable-data-channel-navigation=true` +property. The [demo](www/) implements the protocol and you can easily test this +feature, using the [`wpesrc`] for example. + +As an example, the following pipeline allows you to navigate the GStreamer +documentation inside the video running within your web browser (in +http://127.0.0.1:8000 if you followed previous steps of that readme): + +``` +gst-launch-1.0 wpesrc location=https://gstreamer.freedesktop.org/documentation/ ! webrtcsink enable-data-channel-navigation=true +``` + +[`GstNavigation`]: https://gstreamer.freedesktop.org/documentation/video/gstnavigation.html +[`wpesrc`]: https://gstreamer.freedesktop.org/documentation/wpe/wpesrc.html + +## Testing congestion control + +For the purpose of testing congestion in a reproducible manner, a +[simple tool] has been used, I only used it on Linux but it is documented +as usable on MacOS too. I had to run the client browser on a separate +machine on my local network for congestion to actually be applied, I didn't +look into why that was necessary. + +My testing procedure was: + +* identify the server machine network interface (eg with `ifconfig` on Linux) + +* identify the client machine IP address (eg with `ifconfig` on Linux) + +* start the various services as explained in the Usage section (use + `GST_DEBUG=webrtcsink:7` to get detailed logs about congestion control) + +* start playback in the client browser + +* Run a `comcast` command on the server machine, for instance: + + ``` shell + /home/meh/go/bin/comcast --device=$SERVER_INTERFACE --target-bw 3000 --target-addr=$CLIENT_IP --target-port=1:65535 --target-proto=udp + ``` + +* Observe the bitrate sharply decreasing, playback should slow down briefly + then catch back up + +* Remove the bandwidth limitation, and observe the bitrate eventually increasing + back to a maximum: + + ``` shell + /home/meh/go/bin/comcast --device=$SERVER_INTERFACE --stop + ``` + +For comparison, the congestion control property can be set to disabled on +webrtcsink, then the above procedure applied again, the expected result is +for playback to simply crawl down to a halt until the bandwidth limitation +is lifted: + +``` shell +gst-launch-1.0 webrtcsink congestion-control=disabled +``` + +[simple tool]: https://github.com/tylertreat/comcast + +## Monitoring tool + +An example server / client application for monitoring per-consumer stats +can be found [here]. + +[here]: plugins/examples/README.md + +## License + +All the rust code in this repository is licensed under the [Mozilla Public License Version 2.0]. + +Parts of the JavaScript code in the www/ example are licensed under the [Apache License, Version 2.0], +the rest is licensed under the [Mozilla Public License Version 2.0] unless advertised in the +header. + +[Mozilla Public License Version 2.0]: http://opensource.org/licenses/MPL-2.0 +[Apache License, Version 2.0]: https://www.apache.org/licenses/LICENSE-2.1 diff --git a/net/webrtc/plugins/Cargo.toml b/net/webrtc/plugins/Cargo.toml new file mode 100644 index 00000000..51debdda --- /dev/null +++ b/net/webrtc/plugins/Cargo.toml @@ -0,0 +1,68 @@ +[package] +name = "webrtcsink" +version = "0.1.0" +edition = "2018" +authors = ["Mathieu Duponchelle <mathieu@centricular.com>"] +license = "MPL-2.0" +description = "GStreamer WebRTC sink" +repository = "https://github.com/centricular/webrtcsink/" +build = "build.rs" + +[dependencies] +gst = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer", features = ["v1_20", "serde"] } +gst-app = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-app", features = ["v1_20"] } +gst-video = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-video", features = ["v1_20", "serde"] } +gst-webrtc = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-webrtc", features = ["v1_20"] } +gst-sdp = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-sdp", features = ["v1_20"] } +gst-rtp = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-rtp", features = ["v1_20"] } +gst-utils = { git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", package = "gstreamer-utils" } +once_cell = "1.0" +chrono = { version = "0.4", default-features = false } +smallvec = "1" +anyhow = "1" +thiserror = "1" +futures = "0.3" +async-std = { version = "1", features = ["unstable"] } +async-native-tls = { version = "0.4.0" } +async-tungstenite = { version = "0.17", features = ["async-std-runtime", "async-native-tls"] } +serde = "1" +serde_json = "1" +fastrand = "1.0" +webrtcsink-protocol = { version = "0.1", path="../protocol" } +human_bytes = "0.3.1" + +[dev-dependencies] +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } +tracing-log = "0.1" +uuid = { version = "1", features = ["v4"] } +clap = { version = "4", features = ["derive"] } + +[lib] +name = "webrtcsink" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[build-dependencies] +gst-plugin-version-helper = "0.7" + +[features] +static = [] +capi = [] +gst1_22 = ["gst/v1_22", "gst-app/v1_22", "gst-video/v1_22", "gst-webrtc/v1_22", "gst-sdp/v1_22", "gst-rtp/v1_22"] + +[package.metadata.capi] +min_version = "0.8.0" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-rtp >= 1.20, gstreamer-webrtc >= 1.20, gstreamer-1.0 >= 1.20, gstreamer-app >= 1.20, gstreamer-video >= 1.20, gstreamer-sdp >= 1.20, gobject-2.0, glib-2.0, gmodule-2.0" + +[[example]] +name = "webrtcsink-stats-server" diff --git a/net/webrtc/plugins/build.rs b/net/webrtc/plugins/build.rs new file mode 100644 index 00000000..cda12e57 --- /dev/null +++ b/net/webrtc/plugins/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::info() +} diff --git a/net/webrtc/plugins/examples/README.md b/net/webrtc/plugins/examples/README.md new file mode 100644 index 00000000..7b402e69 --- /dev/null +++ b/net/webrtc/plugins/examples/README.md @@ -0,0 +1,18 @@ +# webrtcsink examples + +Collection (1-sized for now) of webrtcsink examples + +## webrtcsink-stats-server + +A simple application that instantiates a webrtcsink and serves stats +over websockets. + +The application expects a signalling server to be running at `ws://localhost:8443`, +similar to the usage example in the main README. + +``` shell +cargo run --example webrtcsink-stats-server +``` + +Once it is running, follow the instruction in the webrtcsink-stats folder to +run an example client. diff --git a/net/webrtc/plugins/examples/webrtcsink-stats-server.rs b/net/webrtc/plugins/examples/webrtcsink-stats-server.rs new file mode 100644 index 00000000..6088dba4 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats-server.rs @@ -0,0 +1,235 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use anyhow::Error; + +use async_std::net::{TcpListener, TcpStream}; +use async_std::task; +use async_tungstenite::tungstenite::Message as WsMessage; +use clap::Parser; +use futures::channel::mpsc; +use futures::prelude::*; +use gst::glib::Type; +use gst::prelude::*; +use tracing::{debug, info, trace}; +use tracing_subscriber::prelude::*; + +#[derive(Parser, Debug)] +#[clap(about, version, author)] +/// Program arguments +struct Args { + /// URI of file to serve. Must hold at least one audio and video stream + uri: String, + /// Disable Forward Error Correction + #[clap(long)] + disable_fec: bool, + /// Disable retransmission + #[clap(long)] + disable_retransmission: bool, + /// Disable congestion control + #[clap(long)] + disable_congestion_control: bool, +} + +fn serialize_value(val: &gst::glib::Value) -> Option<serde_json::Value> { + match val.type_() { + Type::STRING => Some(val.get::<String>().unwrap().into()), + Type::BOOL => Some(val.get::<bool>().unwrap().into()), + Type::I32 => Some(val.get::<i32>().unwrap().into()), + Type::U32 => Some(val.get::<u32>().unwrap().into()), + Type::I_LONG | Type::I64 => Some(val.get::<i64>().unwrap().into()), + Type::U_LONG | Type::U64 => Some(val.get::<u64>().unwrap().into()), + Type::F32 => Some(val.get::<f32>().unwrap().into()), + Type::F64 => Some(val.get::<f64>().unwrap().into()), + _ => { + if let Ok(s) = val.get::<gst::Structure>() { + serde_json::to_value( + s.iter() + .filter_map(|(name, value)| { + serialize_value(value).map(|value| (name.to_string(), value)) + }) + .collect::<HashMap<String, serde_json::Value>>(), + ) + .ok() + } else if let Ok(a) = val.get::<gst::Array>() { + serde_json::to_value( + a.iter() + .filter_map(|value| serialize_value(value)) + .collect::<Vec<serde_json::Value>>(), + ) + .ok() + } else if let Some((_klass, values)) = gst::glib::FlagsValue::from_value(val) { + Some( + values + .iter() + .map(|value| value.nick()) + .collect::<Vec<&str>>() + .join("+") + .into(), + ) + } else if let Ok(value) = val.serialize() { + Some(value.as_str().into()) + } else { + None + } + } + } +} + +#[derive(Clone)] +struct Listener { + id: uuid::Uuid, + sender: mpsc::Sender<WsMessage>, +} + +struct State { + listeners: Vec<Listener>, +} + +async fn run(args: Args) -> Result<(), Error> { + tracing_log::LogTracer::init().expect("Failed to set logger"); + let env_filter = tracing_subscriber::EnvFilter::try_from_env("WEBRTCSINK_STATS_LOG") + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let fmt_layer = tracing_subscriber::fmt::layer() + .with_thread_ids(true) + .with_target(true) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ); + let subscriber = tracing_subscriber::Registry::default() + .with(env_filter) + .with(fmt_layer); + tracing::subscriber::set_global_default(subscriber).expect("Failed to set subscriber"); + + let state = Arc::new(Mutex::new(State { listeners: vec![] })); + + let addr = "127.0.0.1:8484".to_string(); + + // Create the event loop and TCP listener we'll accept connections on. + let try_socket = TcpListener::bind(&addr).await; + let listener = try_socket.expect("Failed to bind"); + info!("Listening on: {}", addr); + + let pipeline_str = format!( + "webrtcsink name=ws do-retransmission={} do-fec={} congestion-control={} \ + uridecodebin name=d uri={} \ + d. ! video/x-raw ! queue ! ws.video_0 \ + d. ! audio/x-raw ! queue ! ws.audio_0", + !args.disable_retransmission, + !args.disable_fec, + if args.disable_congestion_control { + "disabled" + } else { + "homegrown" + }, + args.uri + ); + + let pipeline = gst::parse_launch(&pipeline_str)?; + let ws = pipeline + .downcast_ref::<gst::Bin>() + .unwrap() + .by_name("ws") + .unwrap(); + + ws.connect("encoder-setup", false, |values| { + let encoder = values[3].get::<gst::Element>().unwrap(); + + info!("Encoder: {}", encoder.factory().unwrap().name()); + + let configured = if let Some(factory) = encoder.factory() { + match factory.name().as_str() { + "does-not-exist" => { + // One could configure a hardware encoder to their liking here, + // and return true to make sure webrtcsink does not do any configuration + // of its own + true + } + _ => false, + } + } else { + false + }; + + Some(configured.to_value()) + }); + + let ws_clone = ws.downgrade(); + let state_clone = state.clone(); + task::spawn(async move { + let mut interval = async_std::stream::interval(std::time::Duration::from_millis(100)); + + while interval.next().await.is_some() { + if let Some(ws) = ws_clone.upgrade() { + let stats = ws.property::<gst::Structure>("stats"); + let stats = serialize_value(&stats.to_value()).unwrap(); + debug!("Stats: {}", serde_json::to_string_pretty(&stats).unwrap()); + let msg = WsMessage::Text(serde_json::to_string(&stats).unwrap()); + + let listeners = state_clone.lock().unwrap().listeners.clone(); + + for mut listener in listeners { + if listener.sender.send(msg.clone()).await.is_err() { + let mut state = state_clone.lock().unwrap(); + let index = state + .listeners + .iter() + .position(|l| l.id == listener.id) + .unwrap(); + state.listeners.remove(index); + } + } + } else { + break; + } + } + }); + + pipeline.set_state(gst::State::Playing)?; + + while let Ok((stream, _)) = listener.accept().await { + task::spawn(accept_connection(state.clone(), stream)); + } + + Ok(()) +} + +async fn accept_connection(state: Arc<Mutex<State>>, stream: TcpStream) { + let addr = stream + .peer_addr() + .expect("connected streams should have a peer address"); + info!("Peer address: {}", addr); + + let mut ws_stream = async_tungstenite::accept_async(stream) + .await + .expect("Error during the websocket handshake occurred"); + + info!("New WebSocket connection: {}", addr); + + let mut state = state.lock().unwrap(); + let (sender, mut receiver) = mpsc::channel::<WsMessage>(1000); + state.listeners.push(Listener { + id: uuid::Uuid::new_v4(), + sender, + }); + drop(state); + + task::spawn(async move { + while let Some(msg) = receiver.next().await { + trace!("Sending to one listener!"); + if ws_stream.send(msg).await.is_err() { + info!("Listener errored out"); + receiver.close(); + } + } + }); +} + +fn main() -> Result<(), Error> { + gst::init()?; + + let args = Args::parse(); + + task::block_on(run(args)) +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/.gitignore b/net/webrtc/plugins/examples/webrtcsink-stats/.gitignore new file mode 100644 index 00000000..126fe84d --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/.gitignore @@ -0,0 +1,4 @@ +/node_modules/ +/dist/ +/.vscode/ +.DS_Store diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/README.md b/net/webrtc/plugins/examples/webrtcsink-stats/README.md new file mode 100644 index 00000000..e1c779bc --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/README.md @@ -0,0 +1,20 @@ +# Example web client for webrtcsink-stats-server + +This web client will display live statistics as received through a +websocket connected to a `webrtcsink-stats-server`. + +Usage: + +``` shell +npm install +npm run dev +``` + +Then navigate to `http://localhost:3000/`. Once consumers are connected +to the webrtc-sink-stats-server, they should be listed on the page, clicking +on any consumer will show a modal with plots for some of the most interesting +statistics. + +The stat server can also be specified through the `remote-url` search parameter, +for example you can access a distant stat server with +`http://localhost:3000?remote-uri=my-remoye.com:72522`. diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/index.html b/net/webrtc/plugins/examples/webrtcsink-stats/index.html new file mode 100644 index 00000000..d2f6839b --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/index.html @@ -0,0 +1,13 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="UTF-8" /> + <link rel="icon" href="/favicon.ico" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>Svelte + TS + Vite App</title> + </head> + <body> + <div id="app"></div> + <script type="module" src="/src/main.ts"></script> + </body> +</html> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/package-lock.json b/net/webrtc/plugins/examples/webrtcsink-stats/package-lock.json new file mode 100644 index 00000000..03d18f6d --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/package-lock.json @@ -0,0 +1,919 @@ +{ + "name": "webrtcsink-stats", + "version": "0.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@fortawesome/fontawesome-common-types": { + "version": "0.2.36", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-0.2.36.tgz", + "integrity": "sha512-a/7BiSgobHAgBWeN7N0w+lAhInrGxksn13uK7231n2m8EDPE3BMCl9NZLTGrj9ZXfCmC6LM0QLqXidIizVQ6yg==", + "dev": true + }, + "@fortawesome/free-solid-svg-icons": { + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-5.15.4.tgz", + "integrity": "sha512-JLmQfz6tdtwxoihXLg6lT78BorrFyCf59SAwBM6qV/0zXyVeDygJVb3fk+j5Qat+Yvcxp1buLTY5iDh1ZSAQ8w==", + "dev": true, + "requires": { + "@fortawesome/fontawesome-common-types": "^0.2.36" + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@rollup/pluginutils": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-4.1.1.tgz", + "integrity": "sha512-clDjivHqWGXi7u+0d2r2sBi4Ie6VLEAzWMIkvJLnDmxoOhBYOTfzGbOQBA32THHm11/LiJbd01tJUpJsbshSWQ==", + "dev": true, + "requires": { + "estree-walker": "^2.0.1", + "picomatch": "^2.2.2" + } + }, + "@sveltejs/vite-plugin-svelte": { + "version": "1.0.0-next.30", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-1.0.0-next.30.tgz", + "integrity": "sha512-YQqdMxjL1VgSFk4/+IY3yLwuRRapPafPiZTiaGEq1psbJYSNYUWx9F1zMm32GMsnogg3zn99mGJOqe3ld3HZSg==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^4.1.1", + "debug": "^4.3.2", + "kleur": "^4.1.4", + "magic-string": "^0.25.7", + "require-relative": "^0.8.7", + "svelte-hmr": "^0.14.7" + } + }, + "@tsconfig/svelte": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@tsconfig/svelte/-/svelte-2.0.1.tgz", + "integrity": "sha512-aqkICXbM1oX5FfgZd2qSSAGdyo/NRxjWCamxoyi3T8iVQnzGge19HhDYzZ6NrVOW7bhcWNSq9XexWFtMzbB24A==", + "dev": true + }, + "@types/node": { + "version": "16.11.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.10.tgz", + "integrity": "sha512-3aRnHa1KlOEEhJ6+CvyHKK5vE9BcLGjtUpwvqYLRvYNQKMfabu3BwfJaA/SLW8dxe28LsNDjtHwePTuzn3gmOA==", + "dev": true + }, + "@types/pug": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.5.tgz", + "integrity": "sha512-LOnASQoeNZMkzexRuyqcBBDZ6rS+rQxUMkmj5A0PkhhiSZivLIuz6Hxyr1mkGoEZEkk66faROmpMi4fFkrKsBA==", + "dev": true + }, + "@types/sass": { + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/@types/sass/-/sass-1.43.1.tgz", + "integrity": "sha512-BPdoIt1lfJ6B7rw35ncdwBZrAssjcwzI5LByIrYs+tpXlj/CAkuVdRsgZDdP4lq5EjyWzwxZCqAoFyHKFwp32g==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", + "dev": true + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "chokidar": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.2.tgz", + "integrity": "sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "detect-indent": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", + "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", + "dev": true + }, + "es6-promise": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz", + "integrity": "sha1-oIzd6EzNvzTQJ6FFG8kdS80ophM=", + "dev": true + }, + "esbuild": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.13.15.tgz", + "integrity": "sha512-raCxt02HBKv8RJxE8vkTSCXGIyKHdEdGfUmiYb8wnabnaEmHzyW7DCHb5tEN0xU8ryqg5xw54mcwnYkC4x3AIw==", + "dev": true, + "requires": { + "esbuild-android-arm64": "0.13.15", + "esbuild-darwin-64": "0.13.15", + "esbuild-darwin-arm64": "0.13.15", + "esbuild-freebsd-64": "0.13.15", + "esbuild-freebsd-arm64": "0.13.15", + "esbuild-linux-32": "0.13.15", + "esbuild-linux-64": "0.13.15", + "esbuild-linux-arm": "0.13.15", + "esbuild-linux-arm64": "0.13.15", + "esbuild-linux-mips64le": "0.13.15", + "esbuild-linux-ppc64le": "0.13.15", + "esbuild-netbsd-64": "0.13.15", + "esbuild-openbsd-64": "0.13.15", + "esbuild-sunos-64": "0.13.15", + "esbuild-windows-32": "0.13.15", + "esbuild-windows-64": "0.13.15", + "esbuild-windows-arm64": "0.13.15" + } + }, + "esbuild-android-arm64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.13.15.tgz", + "integrity": "sha512-m602nft/XXeO8YQPUDVoHfjyRVPdPgjyyXOxZ44MK/agewFFkPa8tUo6lAzSWh5Ui5PB4KR9UIFTSBKh/RrCmg==", + "dev": true, + "optional": true + }, + "esbuild-darwin-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.13.15.tgz", + "integrity": "sha512-ihOQRGs2yyp7t5bArCwnvn2Atr6X4axqPpEdCFPVp7iUj4cVSdisgvEKdNR7yH3JDjW6aQDw40iQFoTqejqxvQ==", + "dev": true, + "optional": true + }, + "esbuild-darwin-arm64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.13.15.tgz", + "integrity": "sha512-i1FZssTVxUqNlJ6cBTj5YQj4imWy3m49RZRnHhLpefFIh0To05ow9DTrXROTE1urGTQCloFUXTX8QfGJy1P8dQ==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.13.15.tgz", + "integrity": "sha512-G3dLBXUI6lC6Z09/x+WtXBXbOYQZ0E8TDBqvn7aMaOCzryJs8LyVXKY4CPnHFXZAbSwkCbqiPuSQ1+HhrNk7EA==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-arm64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.13.15.tgz", + "integrity": "sha512-KJx0fzEDf1uhNOZQStV4ujg30WlnwqUASaGSFPhznLM/bbheu9HhqZ6mJJZM32lkyfGJikw0jg7v3S0oAvtvQQ==", + "dev": true, + "optional": true + }, + "esbuild-linux-32": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.13.15.tgz", + "integrity": "sha512-ZvTBPk0YWCLMCXiFmD5EUtB30zIPvC5Itxz0mdTu/xZBbbHJftQgLWY49wEPSn2T/TxahYCRDWun5smRa0Tu+g==", + "dev": true, + "optional": true + }, + "esbuild-linux-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.13.15.tgz", + "integrity": "sha512-eCKzkNSLywNeQTRBxJRQ0jxRCl2YWdMB3+PkWFo2BBQYC5mISLIVIjThNtn6HUNqua1pnvgP5xX0nHbZbPj5oA==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.13.15.tgz", + "integrity": "sha512-wUHttDi/ol0tD8ZgUMDH8Ef7IbDX+/UsWJOXaAyTdkT7Yy9ZBqPg8bgB/Dn3CZ9SBpNieozrPRHm0BGww7W/jA==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.13.15.tgz", + "integrity": "sha512-bYpuUlN6qYU9slzr/ltyLTR9YTBS7qUDymO8SV7kjeNext61OdmqFAzuVZom+OLW1HPHseBfJ/JfdSlx8oTUoA==", + "dev": true, + "optional": true + }, + "esbuild-linux-mips64le": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.13.15.tgz", + "integrity": "sha512-KlVjIG828uFPyJkO/8gKwy9RbXhCEUeFsCGOJBepUlpa7G8/SeZgncUEz/tOOUJTcWMTmFMtdd3GElGyAtbSWg==", + "dev": true, + "optional": true + }, + "esbuild-linux-ppc64le": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.13.15.tgz", + "integrity": "sha512-h6gYF+OsaqEuBjeesTBtUPw0bmiDu7eAeuc2OEH9S6mV9/jPhPdhOWzdeshb0BskRZxPhxPOjqZ+/OqLcxQwEQ==", + "dev": true, + "optional": true + }, + "esbuild-netbsd-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.13.15.tgz", + "integrity": "sha512-3+yE9emwoevLMyvu+iR3rsa+Xwhie7ZEHMGDQ6dkqP/ndFzRHkobHUKTe+NCApSqG5ce2z4rFu+NX/UHnxlh3w==", + "dev": true, + "optional": true + }, + "esbuild-openbsd-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.13.15.tgz", + "integrity": "sha512-wTfvtwYJYAFL1fSs8yHIdf5GEE4NkbtbXtjLWjM3Cw8mmQKqsg8kTiqJ9NJQe5NX/5Qlo7Xd9r1yKMMkHllp5g==", + "dev": true, + "optional": true + }, + "esbuild-sunos-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.13.15.tgz", + "integrity": "sha512-lbivT9Bx3t1iWWrSnGyBP9ODriEvWDRiweAs69vI+miJoeKwHWOComSRukttbuzjZ8r1q0mQJ8Z7yUsDJ3hKdw==", + "dev": true, + "optional": true + }, + "esbuild-windows-32": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.13.15.tgz", + "integrity": "sha512-fDMEf2g3SsJ599MBr50cY5ve5lP1wyVwTe6aLJsM01KtxyKkB4UT+fc5MXQFn3RLrAIAZOG+tHC+yXObpSn7Nw==", + "dev": true, + "optional": true + }, + "esbuild-windows-64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.13.15.tgz", + "integrity": "sha512-9aMsPRGDWCd3bGjUIKG/ZOJPKsiztlxl/Q3C1XDswO6eNX/Jtwu4M+jb6YDH9hRSUflQWX0XKAfWzgy5Wk54JQ==", + "dev": true, + "optional": true + }, + "esbuild-windows-arm64": { + "version": "0.13.15", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.13.15.tgz", + "integrity": "sha512-zzvyCVVpbwQQATaf3IG8mu1IwGEiDxKkYUdA4FpoCHi1KtPa13jeScYDjlW0Qh+ebWzpKfR2ZwvqAQkSWNcKjA==", + "dev": true, + "optional": true + }, + "estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, + "fast-glob": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.7.tgz", + "integrity": "sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + } + }, + "fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-core-module": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz", + "integrity": "sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "kleur": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.4.tgz", + "integrity": "sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA==", + "dev": true + }, + "magic-string": { + "version": "0.25.7", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz", + "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==", + "dev": true, + "requires": { + "sourcemap-codec": "^1.4.4" + } + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, + "micromatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", + "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "dev": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.2.3" + } + }, + "min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, + "mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "nanoid": { + "version": "3.1.30", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", + "integrity": "sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==", + "dev": true + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "picomatch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", + "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "dev": true + }, + "postcss": { + "version": "8.4.4", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.4.tgz", + "integrity": "sha512-joU6fBsN6EIer28Lj6GDFoC/5yOZzLCfn0zHAn/MYXI7aPt4m4hK5KC5ovEZXy+lnCjmYIbQWngvju2ddyEr8Q==", + "dev": true, + "requires": { + "nanoid": "^3.1.30", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.1" + } + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "require-relative": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz", + "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=", + "dev": true + }, + "resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "dev": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "rollup": { + "version": "2.60.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.60.1.tgz", + "integrity": "sha512-akwfnpjY0rXEDSn1UTVfKXJhPsEBu+imi1gqBA1ZkHGydUnkV/fWCC90P7rDaLEW8KTwBcS1G3N4893Ndz+jwg==", + "dev": true, + "requires": { + "fsevents": "~2.3.2" + } + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "sade": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.7.4.tgz", + "integrity": "sha512-y5yauMD93rX840MwUJr7C1ysLFBgMspsdTo4UVrDg3fXDvtwOyIqykhVAAm6fk/3au77773itJStObgK+LKaiA==", + "dev": true, + "requires": { + "mri": "^1.1.0" + } + }, + "sander": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/sander/-/sander-0.5.1.tgz", + "integrity": "sha1-dB4kXiMfB8r7b98PEzrfohalAq0=", + "dev": true, + "requires": { + "es6-promise": "^3.1.2", + "graceful-fs": "^4.1.3", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.2" + } + }, + "sass": { + "version": "1.43.5", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.43.5.tgz", + "integrity": "sha512-WuNm+eAryMgQluL7Mbq9M4EruyGGMyal7Lu58FfnRMVWxgUzIvI7aSn60iNt3kn5yZBMR7G84fAGDcwqOF5JOg==", + "dev": true, + "requires": { + "chokidar": ">=3.0.0 <4.0.0" + } + }, + "sorcery": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/sorcery/-/sorcery-0.10.0.tgz", + "integrity": "sha1-iukK19fLBfxZ8asMY3hF1cFaUrc=", + "dev": true, + "requires": { + "buffer-crc32": "^0.2.5", + "minimist": "^1.2.0", + "sander": "^0.5.0", + "sourcemap-codec": "^1.3.0" + } + }, + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "dev": true + }, + "source-map-js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.1.tgz", + "integrity": "sha512-4+TN2b3tqOCd/kaGRJ/sTYA0tR0mdXx26ipdolxcwtJVqEnqNYvlCAt1q3ypy4QMlYus+Zh34RNtYLoq2oQ4IA==", + "dev": true + }, + "sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", + "dev": true + }, + "strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "requires": { + "min-indent": "^1.0.0" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "svelte": { + "version": "3.44.2", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-3.44.2.tgz", + "integrity": "sha512-jrZhZtmH3ZMweXg1Q15onb8QlWD+a5T5Oca4C1jYvSURp2oD35h4A5TV6t6MEa93K4LlX6BkafZPdQoFjw/ylA==", + "dev": true + }, + "svelte-check": { + "version": "2.2.10", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-2.2.10.tgz", + "integrity": "sha512-UVLd/N7hUIG2v6dytofsw8MxYn2iS2hpNSglsGz9Z9b8ZfbJ5jayl4Mm1SXhNwiFs5aklG90zSBJtd7NTK8dTg==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "chokidar": "^3.4.1", + "fast-glob": "^3.2.7", + "import-fresh": "^3.2.1", + "minimist": "^1.2.5", + "sade": "^1.7.4", + "source-map": "^0.7.3", + "svelte-preprocess": "^4.0.0", + "typescript": "*" + } + }, + "svelte-fa": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/svelte-fa/-/svelte-fa-2.4.0.tgz", + "integrity": "sha512-0bnbMGbsE1LUnlioDcf27tl2O8kjuXlTXMXzIxC7LoIOWmqn0D+zd539HfLiQbdLuOHGTaynwN9V+4ehhEu1Jw==", + "dev": true + }, + "svelte-hmr": { + "version": "0.14.7", + "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.14.7.tgz", + "integrity": "sha512-pDrzgcWSoMaK6AJkBWkmgIsecW0GChxYZSZieIYfCP0v2oPyx2CYU/zm7TBIcjLVUPP714WxmViE9Thht4etog==", + "dev": true + }, + "svelte-preprocess": { + "version": "4.9.8", + "resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-4.9.8.tgz", + "integrity": "sha512-EQS/oRZzMtYdAprppZxY3HcysKh11w54MgA63ybtL+TAZ4hVqYOnhw41JVJjWN9dhPnNjjLzvbZ2tMhTsla1Og==", + "dev": true, + "requires": { + "@types/pug": "^2.0.4", + "@types/sass": "^1.16.0", + "detect-indent": "^6.0.0", + "magic-string": "^0.25.7", + "sorcery": "^0.10.0", + "strip-indent": "^3.0.0" + } + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, + "typescript": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.2.tgz", + "integrity": "sha512-5BlMof9H1yGt0P8/WF+wPNw6GfctgGjXp5hkblpyT+8rkASSmkUKMXrxR0Xg8ThVCi/JnHQiKXeBaEwCeQwMFw==", + "dev": true + }, + "vite": { + "version": "2.6.14", + "resolved": "https://registry.npmjs.org/vite/-/vite-2.6.14.tgz", + "integrity": "sha512-2HA9xGyi+EhY2MXo0+A2dRsqsAG3eFNEVIo12olkWhOmc8LfiM+eMdrXf+Ruje9gdXgvSqjLI9freec1RUM5EA==", + "dev": true, + "requires": { + "esbuild": "^0.13.2", + "fsevents": "~2.3.2", + "postcss": "^8.3.8", + "resolve": "^1.20.0", + "rollup": "^2.57.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + } + } +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/package.json b/net/webrtc/plugins/examples/webrtcsink-stats/package.json new file mode 100644 index 00000000..b0f3947e --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/package.json @@ -0,0 +1,27 @@ +{ + "name": "webrtcsink-stats", + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "serve": "vite preview", + "check": "svelte-check --tsconfig ./tsconfig.json" + }, + "devDependencies": { + "@fortawesome/free-solid-svg-icons": "^5.15.4", + "@sveltejs/vite-plugin-svelte": "^1.0.0-next.11", + "@tsconfig/svelte": "^2.0.1", + "sass": "^1.43.5", + "svelte": "^3.37.0", + "svelte-check": "^2.1.0", + "svelte-fa": "^2.4.0", + "svelte-preprocess": "^4.7.2", + "tslib": "^2.2.0", + "typescript": "^4.3.2", + "vite": "^2.6.4" + }, + "dependencies": { + "moment": "^2.29.1" + } +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/public/favicon.ico b/net/webrtc/plugins/examples/webrtcsink-stats/public/favicon.ico Binary files differnew file mode 100644 index 00000000..d75d248e --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/public/favicon.ico diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/App.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/App.svelte new file mode 100644 index 00000000..48fcc727 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/App.svelte @@ -0,0 +1,182 @@ +<svelte:head> + <script src="https://cdn.plot.ly/plotly-latest.min.js" type="text/javascript"></script> +</svelte:head> + +<script lang="ts"> + import Home from '@/pages/Home.svelte' + import Header from '@/components/Header.svelte' + import type { ConsumerType } from '@/types/app' + import { WebSocketStatus, MitigationMode } from '@/types/app' + import { onMount, onDestroy } from 'svelte'; + + let ws: WebSocket | undefined = undefined + let websocketStatus: WebSocketStatus = WebSocketStatus.Connecting + let consumers: Map<string, ConsumerType> = new Map () + let consumers_array: Array<ConsumerType> = [] + let timeout: ReturnType<typeof setTimeout> | undefined = undefined + + const updateConsumerStats = (consumer: ConsumerType, stats: Object) => { + let target_bitrate = 0 + let fec_percentage = 0 + let keyframe_requests = 0 + let retransmission_requests = 0 + let bitrate_sent = 0 + let bitrate_recv = 0 + let packet_loss = 0 + let delta_of_delta = 0 + + if (stats["consumer-stats"]["video-encoders"].length > 0) { + let venc = stats["consumer-stats"]["video-encoders"][0] + target_bitrate = venc["bitrate"] + fec_percentage = venc["fec-percentage"] + consumer.video_codec = venc["codec-name"] + + let mitigation_mode = MitigationMode.None + + for (let mode of venc["mitigation-mode"].split("+")) { + switch (mode) { + case "none": { + mitigation_mode |= MitigationMode.None + break + } + case "downscaled": { + mitigation_mode |= MitigationMode.Downscaled + break + } + case "downsampled": { + mitigation_mode |= MitigationMode.Downsampled + break + } + } + } + + consumer.mitigation_mode = mitigation_mode + } + + + for (let svalue of Object.values(stats)) { + if (svalue["type"] == "transport") { + let twcc_stats = svalue["gst-twcc-stats"] + if (twcc_stats !== undefined) { + bitrate_sent = twcc_stats["bitrate-sent"] + bitrate_recv = twcc_stats["bitrate-recv"] + packet_loss = twcc_stats["packet-loss-pct"] + delta_of_delta = twcc_stats["avg-delta-of-delta"] + } + } else if (svalue["type"] == "outbound-rtp") { + keyframe_requests += svalue["pli-count"] + retransmission_requests += svalue["nack-count"] + } + } + + consumer.stats["target_bitrate"] = target_bitrate + consumer.stats["fec_percentage"] = fec_percentage + consumer.stats["bitrate_sent"] = bitrate_sent + consumer.stats["bitrate_recv"] = bitrate_recv + consumer.stats["packet_loss"] = packet_loss + consumer.stats["delta_of_delta"] = delta_of_delta + consumer.stats["keyframe_requests"] = keyframe_requests + consumer.stats["retransmission_requests"] = retransmission_requests + } + + const fetchStats = () => { + const urlParams = new URLSearchParams(window.location.search); + var remote_server = urlParams.get('remote-url'); + if (!remote_server) + remote_server = "127.0.0.1:8484" + const ws_url = `ws://${remote_server}`; + + console.info(`Logging to ${ws_url}`); + ws = new WebSocket(ws_url); + + ws.onerror = () => { + websocketStatus = WebSocketStatus.Error + } + + ws.onclose = () => { + websocketStatus = WebSocketStatus.Error + consumers = new Map() + consumers_array = [] + timeout = setTimeout(fetchStats, 500) + } + + ws.onopen = () => { + websocketStatus = WebSocketStatus.Connected + } + + ws.onmessage = (event) => { + let stats = JSON.parse(event.data) + // Set is supposed to be buildable from an iterator, + // no idea why the Arra.from is needed .. + let to_remove = new Set(Array.from(consumers.keys())) + + for (let [key, value] of Object.entries(stats)) { + let consumer = undefined; + + if (consumers.get(key) === undefined) { + consumer = { + id: key, + video_codec: undefined, + mitigation_mode: MitigationMode.None, + stats: new Map([ + ["target_bitrate", 0], + ["fec_percentage", 0], + ["bitrate_sent", 0], + ["bitrate_recv", 0], + ["packet_loss", 0], + ["delta_of_delta", 0], + ["keyframe_requests", 0], + ["retransmission_requests", 0], + ]), + } + consumers.set(key, consumer) + } else { + consumer = consumers.get(key) + } + + updateConsumerStats(consumer, value) + + to_remove.delete(key) + } + + for (let key of to_remove) { + consumers.delete(key) + } + + consumers_array = Array.from(consumers.values()) + } + } + + const closeWebSocket = () => { + if (ws != undefined) { + ws.close(); + ws = undefined; + } + + if (timeout != undefined) { + clearTimeout(timeout) + timeout = undefined + } + } + + onMount(fetchStats) + onDestroy(closeWebSocket) +</script> + +<Header websocketStatus={ websocketStatus } /> + +<Home consumers={ consumers_array } /> + +<style lang="scss"> + :root { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, + Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; + height: 100%; + } + :global(body) { + /* this will apply to <body> */ + margin: 0; + height: 100%; + background-color: #fbfbfb; + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/h264.png b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/h264.png Binary files differnew file mode 100644 index 00000000..ab0cd852 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/h264.png diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/svelte.png b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/svelte.png Binary files differnew file mode 100644 index 00000000..e673c91c --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/svelte.png diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp8.png b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp8.png Binary files differnew file mode 100644 index 00000000..f9923be0 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp8.png diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp9.png b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp9.png Binary files differnew file mode 100644 index 00000000..1fcfffff --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/assets/vp9.png diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Consumer.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Consumer.svelte new file mode 100644 index 00000000..7dd1c0a3 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Consumer.svelte @@ -0,0 +1,32 @@ +<script lang="ts"> + import type { ConsumerType } from '@/types/app' + import EncoderProps from '@/components/EncoderProps.svelte' + + export let consumer: ConsumerType + + $: id = consumer.id +</script> + +<div class="consumer-card" on:click > + <div class="id">{id}</div> + <EncoderProps + consumer={consumer} + /> +</div> + +<style lang="scss"> + .consumer-card { + word-break: break-all; + width: 150px; + height: 100px; + margin-right: 15px; + background-color: #fff; + padding: 15px; + border-radius: 10px; + box-shadow: rgb(0 0 0 / 24%) 0px 3px 8px; + .id { + font-weight: bold; + margin-bottom: 10px; + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/components/EncoderProps.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/EncoderProps.svelte new file mode 100644 index 00000000..f6712d8e --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/EncoderProps.svelte @@ -0,0 +1,53 @@ +<script lang="ts"> + import type { ConsumerType } from '@/types/app' + import { MitigationMode } from '@/types/app' + import Fa from 'svelte-fa' + import { faExclamationTriangle, faCheckCircle } from '@fortawesome/free-solid-svg-icons'; + import vp8_logo from '@/assets/vp8.png' + import vp9_logo from '@/assets/vp9.png' + import h264_logo from '@/assets/h264.png' + + export let consumer: ConsumerType + + $: video_codec = consumer.video_codec + $: mitigation_mode = consumer.mitigation_mode +</script> + +<div class="encoder-props"> + <div class="codec"> + {#if video_codec == "video/x-vp8"} + <img src={vp8_logo} alt="VP8"> + {:else if video_codec == "video/x-vp9"} + <img src={vp9_logo} alt="VP8"> + {:else if video_codec == "video/x-h264"} + <img src={h264_logo} alt="VP8"> + {/if} + </div> + <div> + {#if mitigation_mode & MitigationMode.Downsampled && mitigation_mode & MitigationMode.Downscaled} + <abbr title="Very congested link, video is downscaled and downsampled"> + <Fa icon={faExclamationTriangle} color="tomato" /> + </abbr> + {:else if mitigation_mode & MitigationMode.Downscaled} + <abbr title="Congested link, video is downscaled"> + <Fa icon={faExclamationTriangle} color="orange" /> + </abbr> + {:else} + <abbr title="Link with minimal to no congestion"> + <Fa icon={faCheckCircle} color="lightseagreen" /> + </abbr> + {/if} + </div> +</div> + +<style lang="scss"> + .encoder-props { + display: flex; + justify-content: space-evenly; + .codec { + img { + width: 25px; + } + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Header.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Header.svelte new file mode 100644 index 00000000..4dd4746a --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Header.svelte @@ -0,0 +1,48 @@ +<script lang="ts"> + import { WebSocketStatus } from '@/types/app' + + import logo from '@/assets/svelte.png' + import Fa from 'svelte-fa' + import { faSpinner, faExclamationTriangle, faCheckCircle } from '@fortawesome/free-solid-svg-icons'; + + export let websocketStatus: WebSocketStatus +</script> + +<header class="global-header"> + <div class="app-name"> + <img src={logo} alt="Svelte Logo" class="logo"/> + <div class="title">WebRTC Stats App</div> + <div> + {#if websocketStatus == WebSocketStatus.Connected} + <Fa icon={faCheckCircle} color="lightseagreen" size="1x" /> + {:else if websocketStatus == WebSocketStatus.Connecting} + <Fa icon={faSpinner} color="#afaeae" size="1x" spin /> + {:else if websocketStatus == WebSocketStatus.Error} + <Fa icon={faExclamationTriangle} color="tomato" size="1x" /> + {/if} + </div> + </div> +</header> + +<style lang="scss"> + .global-header { + background-color: #313131; + height: 56px; + color: white; + padding: 15px; + box-sizing: border-box; + .app-name { + display: flex; + align-items: center; + height: 100%; + .logo { + height: 100%; + margin-right: 5px; + } + .title { + font-weight: bold; + margin-right: 5px; + } + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Modal.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Modal.svelte new file mode 100644 index 00000000..b4c7e881 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/Modal.svelte @@ -0,0 +1,50 @@ +<script lang="ts"> + import Fa from 'svelte-fa' + import { faTimes } from '@fortawesome/free-solid-svg-icons'; + import { createEventDispatcher } from 'svelte'; + + const dispatch = createEventDispatcher(); +</script> + +<div class="modal-overlay"> + <div class="modal"> + <div class="modal-header"> + <slot name="title"></slot> + <div class="close-icon" on:click="{() => dispatch('closeModal')}"> + <Fa icon={faTimes} color="#afaeae" size="1x" /> + </div> + </div> + + <slot name="body"></slot> + + <slot name="footer"></slot> + </div> +</div> + +<style lang="scss"> + .modal { + background-color: white; + padding: 15px 0; + border-radius: 10px; + box-shadow: rgb(0 0 0 / 24%) 0px 3px 8px; + &-overlay { + position: absolute; + top: 0; + height: 100vh; + width: 100vw; + display: flex; + align-items: center; + justify-content: center; + background-color: rgb(0, 0, 0, 0.4); + } + &-header { + display: flex; + justify-content: space-between; + padding: 0 15px 10px; + border-bottom: 3px solid #e2e2e2; + } + .close-icon { + cursor: pointer; + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/components/PlotConsumerModal.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/PlotConsumerModal.svelte new file mode 100644 index 00000000..c15ee13f --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/components/PlotConsumerModal.svelte @@ -0,0 +1,142 @@ +<svelte:head> + <script src="https://cdn.plot.ly/plotly-latest.min.js" type="text/javascript"></script> +</svelte:head> + +<script lang="ts"> + import { createEventDispatcher, onMount, onDestroy } from 'svelte'; + import Modal from '@/components/Modal.svelte' + import type { ConsumerType } from '@/types/app' + import EncoderProps from '@/components/EncoderProps.svelte' + + export let consumer: ConsumerType + + $: if (consumer === undefined) { + dispatch('close') + } + + $: id = consumer !== undefined ? consumer.id : undefined + + const dispatch = createEventDispatcher(); + let interval: ReturnType<typeof setInterval> | undefined = undefined + + onMount(() => { + let plotDiv = document.getElementById('plotDiv'); + + let traces = [] + let layout = { + legend: {traceorder: 'reversed'}, + height: 800, + } + let ctr = 1; + let domain_step = 1.0 / consumer.stats.size + let domain_margin = 0.05 + + for (let key of consumer.stats.keys()) { + let trace = { + x: [], + y: [], + xaxis: 'x' + ctr, + yaxis: 'y' + ctr, + mode: 'lines', + line: {shape: 'spline'}, + name: key + } + + traces.push(trace) + + layout['xaxis' + ctr] = { + type: 'date', + } + + layout['yaxis' + ctr] = { + domain: [(ctr - 1) * domain_step, (ctr * domain_step) - domain_margin], + rangemode: "tozero", + } + + ctr += 1 + } + + Plotly.newPlot(plotDiv, traces, layout); + + interval = setInterval(function() { + let time = new Date() + let ctr = 0 + let traces = [] + let data_update = { + x: [], + y: [], + } + + for (let value of Object.values(consumer.stats)) { + data_update.x.push([time]) + data_update.y.push([value]) + traces.push(ctr) + ctr += 1 + } + + Plotly.extendTraces(plotDiv, data_update, traces, 600) + + }, 50); + }); + + onDestroy(() => { + console.log ("destroyed") + + if (interval !== undefined ) { + clearInterval (interval) + interval = undefined + } + }) +</script> + +<Modal on:closeModal="{() => dispatch('close')}"> + <div slot="body" class="modal-body"> + <div class="id">{id}</div> + <EncoderProps + consumer={consumer} + /> + <div id="plotDiv"></div> + </div> + + <div slot="footer" class="modal-footer"> + <div class="buttons-wrapper"> + <button + class="button" + on:click|stopPropagation="{() => dispatch('close')}" + > + Cancel + </button> + </div> + </div> +</Modal> + +<style lang="scss"> + .modal { + &-body { + width: 1000px; + padding: 20px 15px 10px; + gap: 15px 0; + .id { + font-weight: bold; + margin-bottom: 10px; + } + } + &-footer { + padding: 0 15px; + .buttons-wrapper { + text-align: right; + } + .button { + height: 30px; + padding: 0 10px; + text-align: center; + box-sizing: content-box; + border-radius: 3px; + border: 1px solid #000; + &:active { + background-color: #b9b7b7; + } + } + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/main.ts b/net/webrtc/plugins/examples/webrtcsink-stats/src/main.ts new file mode 100644 index 00000000..d8200ac4 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/main.ts @@ -0,0 +1,7 @@ +import App from './App.svelte' + +const app = new App({ + target: document.getElementById('app') +}) + +export default app diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/pages/Home.svelte b/net/webrtc/plugins/examples/webrtcsink-stats/src/pages/Home.svelte new file mode 100644 index 00000000..44808535 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/pages/Home.svelte @@ -0,0 +1,64 @@ +<script lang="ts"> + import type { ConsumerType } from '@/types/app' + import Consumer from '@/components/Consumer.svelte' + import PlotConsumerModal from '@/components/PlotConsumerModal.svelte' + + export let consumers: Array<ConsumerType> + + let consumerToPlot: ConsumerType | undefined + let showPlotModal = false + + /** + * Display the Plot modal + * + * @param {ConsumerType} consumer + */ + const openPlotConsumer = (consumer: ConsumerType) => { + consumerToPlot = consumer + showPlotModal = true + } + + /** + * Close the Plot modal + * + */ + const closePlotConsumer = () => { + consumerToPlot = undefined + showPlotModal = false + } +</script> + +<main> + <div class="consumer-card-container"> + {#each consumers as consumer} + <Consumer + consumer = {consumer} + on:click="{() => { openPlotConsumer(consumer) }}" + /> + {/each} + </div> +</main> + +{#if showPlotModal} + <PlotConsumerModal + consumer={consumers.find(consumer => consumer == consumerToPlot)} + on:close="{closePlotConsumer}" + /> +{/if} + +<style lang="scss"> + main { + padding: 2em; + margin: 0 auto; + width: 100vw; + box-sizing: border-box; + } + .consumer-card { + &-container { + display: flex; + flex-wrap: wrap; + row-gap: 20px; + justify-content: space-evenly; + } + } +</style> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/types/app.ts b/net/webrtc/plugins/examples/webrtcsink-stats/src/types/app.ts new file mode 100644 index 00000000..e34b8c5b --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/types/app.ts @@ -0,0 +1,18 @@ +export enum MitigationMode { + None = 0, + Downscaled = 1, + Downsampled = 2, +} + +export interface ConsumerType { + id: string, + video_codec: string | undefined, + mitigation_mode: MitigationMode, + stats: Map<string, number>, +} + +export enum WebSocketStatus { + Connecting = 0, + Connected = 1, + Error = 2, +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/src/vite-env.d.ts b/net/webrtc/plugins/examples/webrtcsink-stats/src/vite-env.d.ts new file mode 100644 index 00000000..4078e747 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/src/vite-env.d.ts @@ -0,0 +1,2 @@ +/// <reference types="svelte" /> +/// <reference types="vite/client" /> diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/svelte.config.js b/net/webrtc/plugins/examples/webrtcsink-stats/svelte.config.js new file mode 100644 index 00000000..e8be9c97 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/svelte.config.js @@ -0,0 +1,13 @@ +import sveltePreprocess from 'svelte-preprocess' +import * as sass from 'sass' + +export default { + // Consult https://github.com/sveltejs/svelte-preprocess + // for more information about preprocessors + preprocess: sveltePreprocess({ + sass: { + sync: true, + implementation: sass, + }, + }) +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/tsconfig.json b/net/webrtc/plugins/examples/webrtcsink-stats/tsconfig.json new file mode 100644 index 00000000..b8e13491 --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "@tsconfig/svelte/tsconfig.json", + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "node", + "resolveJsonModule": true, + "baseUrl": ".", + /** + * Typechecking JS in `.svelte` and `.js` files by default. + * Disable checkJs if you'd like to use dynamic types in JS. + * Note that setting allowJs false does not prevent the use + * of JS in `.svelte` files. + */ + "allowJs": true, + "checkJs": true, + "paths": { + "@/*": [ + "src/*" + ], + } + }, + "include": ["src/**/*.d.ts", "src/**/*.ts", "src/**/*.js", "src/**/*.svelte"] +} diff --git a/net/webrtc/plugins/examples/webrtcsink-stats/vite.config.js b/net/webrtc/plugins/examples/webrtcsink-stats/vite.config.js new file mode 100644 index 00000000..67694a8a --- /dev/null +++ b/net/webrtc/plugins/examples/webrtcsink-stats/vite.config.js @@ -0,0 +1,13 @@ +import { defineConfig } from 'vite' +import { svelte } from '@sveltejs/vite-plugin-svelte' +import path from 'path'; + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [svelte()], + resolve: { + alias: { + '@': path.resolve('/src'), + }, + } +}) diff --git a/net/webrtc/plugins/src/gcc/imp.rs b/net/webrtc/plugins/src/gcc/imp.rs new file mode 100644 index 00000000..72fe8426 --- /dev/null +++ b/net/webrtc/plugins/src/gcc/imp.rs @@ -0,0 +1,1370 @@ +///! Implements https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02 +///! +///! This element implements the pacing as describe in the spec by running its +///! own streaming thread on its srcpad. It implements the mathematic as closely +///! to the specs as possible and sets the #rtpgccbwe:estimated-bitrate property +///! each time a new estimate is produced. User should connect to the +///! `rtpgccbwe::notify::estimated-bitrate` signal to make the encoders target +///! that new estimated bitrate (the overall target bitrate of the potentially +///! multiple encore should match that target bitrate, the application is +///! responsible for determining what bitrate to give to each encode) +use chrono::Duration; +use gst::{ + glib::{self}, + prelude::*, + subclass::prelude::*, +}; +use once_cell::sync::Lazy; +use std::{ + collections::{BTreeMap, VecDeque}, + fmt, + fmt::Debug, + mem, + sync::Mutex, + time, +}; + +type Bitrate = u32; + +const DEFAULT_MIN_BITRATE: Bitrate = 1000; +const DEFAULT_ESTIMATED_BITRATE: Bitrate = 2_048_000; +const DEFAULT_MAX_BITRATE: Bitrate = 8_192_000; + +static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { + gst::DebugCategory::new( + "gcc", + gst::DebugColorFlags::empty(), + Some("Google Congestion Controller based bandwidth estimator"), + ) +}); + +// Table1. Time limit in milliseconds between packet bursts which identifies a group +static BURST_TIME: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(5)); + +// Table1. Coefficient used for the measured noise variance +// [0.1,0.001] +const CHI: f64 = 0.01; +const ONE_MINUS_CHI: f64 = 1. - CHI; + +// Table1. State noise covariance matrix +const Q: f64 = 0.001; + +// Table1. Initial value for the adaptive threshold +static INITIAL_DEL_VAR_TH: Lazy<Duration> = Lazy::new(|| Duration::microseconds(12500)); + +// Table1. Initial value of the system error covariance +const INITIAL_ERROR_COVARIANCE: f64 = 0.1; + +// Table1. Time required to trigger an overuse signal +static OVERUSE_TIME_TH: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(10)); + +// from 5.5 "beta is typically chosen to be in the interval [0.8, 0.95], 0.85 is the RECOMMENDED value." +const BETA: f64 = 0.85; + +// From "5.5 Rate control" It is RECOMMENDED to measure this average and +// standard deviation with an exponential moving average with the smoothing +// factor 0.5 (NOTE: the spec mentions 0.95 here but in the equations it is 0.5 +// and other implementations use 0.5), as it is expected that this average +// covers multiple occasions at which we are in the Decrease state. +const MOVING_AVERAGE_SMOOTHING_FACTOR: f64 = 0.5; + +// `N(i)` is the number of packets received the past T seconds and `L(j)` is +// the payload size of packet j. A window between 0.5 and 1 second is +// RECOMMENDED. +static PACKETS_RECEIVED_WINDOW: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(1000)); // ms + +// from "5.4 Over-use detector" -> +// Moreover, del_var_th(i) SHOULD NOT be updated if this condition +// holds: +// +// ``` +// |m(i)| - del_var_th(i) > 15 +// ``` +static MAX_M_MINUS_DEL_VAR_TH: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(15)); + +// from 5.4 "It is also RECOMMENDED to clamp del_var_th(i) to the range [6, 600]" +static MIN_THRESHOLD: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(6)); +static MAX_THRESHOLD: Lazy<Duration> = Lazy::new(|| Duration::milliseconds(600)); + +// From 5.5 ""Close" is defined as three standard deviations around this average" +const STANDARD_DEVIATION_CLOSE_NUM: f64 = 3.; + +// Minimal duration between 2 updates on the lost based rate controller +static LOSS_UPDATE_INTERVAL: Lazy<time::Duration> = Lazy::new(|| time::Duration::from_millis(200)); +static LOSS_DECREASE_THRESHOLD: f64 = 0.1; +static LOSS_INCREASE_THRESHOLD: f64 = 0.02; +static LOSS_INCREASE_FACTOR: f64 = 1.05; + +// Minimal duration between 2 updates on the lost based rate controller +static DELAY_UPDATE_INTERVAL: Lazy<time::Duration> = Lazy::new(|| time::Duration::from_millis(100)); + +static ROUND_TRIP_TIME_WINDOW_SIZE: usize = 100; + +fn ts2dur(t: gst::ClockTime) -> Duration { + Duration::nanoseconds(t.nseconds() as i64) +} + +fn dur2ts(t: Duration) -> gst::ClockTime { + gst::ClockTime::from_nseconds(t.num_nanoseconds().unwrap() as u64) +} + +#[derive(Debug)] +enum BandwidthEstimationOp { + /// Don't update target bitrate + Hold, + /// Decrease target bitrate + Decrease(String /* reason */), + Increase(String /* reason */), +} + +#[derive(Debug, Clone, Copy)] +enum ControllerType { + // Running the "delay-based controller" + Delay, + // Running the "loss based controller" + Loss, +} + +#[derive(Debug, Clone, Copy)] +struct Packet { + departure: Duration, + arrival: Duration, + size: usize, + seqnum: u64, +} + +fn human_kbits<T: Into<f64>>(bits: T) -> String { + format!("{:.2}kb", (bits.into() / 1_000.)) +} + +impl Packet { + fn from_structure(structure: &gst::StructureRef) -> Option<Self> { + let lost = structure.get::<bool>("lost").unwrap(); + let departure = match structure.get::<gst::ClockTime>("local-ts") { + Err(e) => { + gst::fixme!( + CAT, + "Got packet feedback without local-ts: {:?} - what does that mean?", + e + ); + return None; + } + Ok(ts) => ts, + }; + + let seqnum = structure.get::<u32>("seqnum").unwrap() as u64; + if lost { + return Some(Packet { + arrival: Duration::zero(), + departure: ts2dur(departure), + size: structure.get::<u32>("size").unwrap() as usize, + seqnum, + }); + } + + let arrival = structure.get::<gst::ClockTime>("remote-ts").unwrap(); + + Some(Packet { + arrival: ts2dur(arrival), + departure: ts2dur(departure), + size: structure.get::<u32>("size").unwrap() as usize, + seqnum, + }) + } +} + +#[derive(Clone)] +struct PacketGroup { + packets: Vec<Packet>, + departure: Duration, // ms + arrival: Option<Duration>, // ms +} + +impl Default for PacketGroup { + fn default() -> Self { + Self { + packets: Default::default(), + departure: Duration::zero(), + arrival: None, + } + } +} + +fn pdur(d: &Duration) -> String { + let stdd = time::Duration::from_nanos(d.num_nanoseconds().unwrap().abs() as u64); + + format!("{}{stdd:?}", if d.lt(&Duration::zero()) { "-" } else { "" }) +} + +impl PacketGroup { + fn add(&mut self, packet: Packet) { + if self.departure.is_zero() { + self.departure = packet.departure; + } + + self.arrival = Some( + self.arrival + .map_or_else(|| packet.arrival, |v| Duration::max(v, packet.arrival)), + ); + self.packets.push(packet); + } + + /// Returns the delta between self.arrival_time and @prev_group.arrival_time in ms + // t(i) - t(i-1) + fn inter_arrival_time(&self, prev_group: &Self) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + self.arrival.unwrap() - prev_group.arrival.unwrap() + } + + fn inter_arrival_time_pkt(&self, next_pkt: &Packet) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + next_pkt.arrival - self.arrival.unwrap() + } + + /// Returns the delta between self.departure_time and @prev_group.departure_time in ms + // T(i) - T(i-1) + fn inter_departure_time(&self, prev_group: &Self) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + self.departure - prev_group.departure + } + + fn inter_departure_time_pkt(&self, next_pkt: &Packet) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + next_pkt.departure - self.departure + } + + /// Returns the delta between intern arrival time and inter departure time in ms + fn inter_delay_variation(&self, prev_group: &Self) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + self.inter_arrival_time(prev_group) - self.inter_departure_time(prev_group) + } + + fn inter_delay_variation_pkt(&self, next_pkt: &Packet) -> Duration { + // Should never be called if we haven't gotten feedback for all + // contained packets + self.inter_arrival_time_pkt(next_pkt) - self.inter_departure_time_pkt(next_pkt) + } +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +enum NetworkUsage { + Normal, + Over, + Under, +} + +struct Detector { + group: PacketGroup, // Packet group that is being filled + prev_group: Option<PacketGroup>, // Group that is ready to be used once "group" is filled + measure: Duration, // Delay variation measure + + last_received_packets: BTreeMap<u64, Packet>, // Order by seqnums, front is the newest, back is the oldest + + // Last loss update + last_loss_update: Option<time::Instant>, + // Moving average of the packet loss + loss_average: f64, + + // Kalman filter fields + gain: f64, + measurement_uncertainty: f64, // var_v_hat(i-1) + estimate_error: f64, // e(i-1) + estimate: Duration, // m_hat(i-1) + + // Threshold fields + threshold: Duration, + last_threshold_update: Option<time::Instant>, + num_deltas: i64, + + // Overuse related fields + increasing_counter: u32, + last_overuse_estimate: Duration, + last_use_detector_update: time::Instant, + increasing_duration: Duration, + + // round-trip-time estimations + rtts: VecDeque<Duration>, + clock: gst::Clock, + + // Current network usage state + usage: NetworkUsage, + + twcc_extended_seqnum: u64, +} + +// Monitors packet loss and network overuse through because of delay +impl Debug for Detector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Network Usage: {:?}. Effective bitrate: {}ps - Measure: {} Estimate: {} threshold {} - overuse_estimate {}", + self.usage, + human_kbits(self.effective_bitrate()), + pdur(&self.measure), + pdur(&self.estimate), + pdur(&self.threshold), + pdur(&self.last_overuse_estimate), + ) + } +} + +impl Detector { + fn new() -> Self { + Detector { + group: Default::default(), + prev_group: Default::default(), + measure: Duration::zero(), + + /* Smallish value to hold PACKETS_RECEIVED_WINDOW packets */ + last_received_packets: BTreeMap::new(), + + last_loss_update: None, + loss_average: 0., + + gain: 0., + measurement_uncertainty: 0., + estimate_error: INITIAL_ERROR_COVARIANCE, + estimate: Duration::zero(), + + threshold: *INITIAL_DEL_VAR_TH, + last_threshold_update: None, + num_deltas: 0, + + last_use_detector_update: time::Instant::now(), + increasing_counter: 0, + last_overuse_estimate: Duration::zero(), + increasing_duration: Duration::zero(), + + rtts: Default::default(), + clock: gst::SystemClock::obtain(), + + usage: NetworkUsage::Normal, + + twcc_extended_seqnum: 0, + } + } + + fn loss_ratio(&self) -> f64 { + self.loss_average + } + + fn update_last_received_packets(&mut self, packet: Packet) { + self.last_received_packets.insert(packet.seqnum, packet); + self.evict_old_received_packets(); + } + + fn evict_old_received_packets(&mut self) { + let last_arrival = self + .last_received_packets + .values() + .next_back() + .unwrap() + .arrival; + + while last_arrival - self.oldest_packet_in_window_ts() > *PACKETS_RECEIVED_WINDOW { + let oldest_seqnum = self.last_received_packets.iter().next().unwrap().0.clone(); + self.last_received_packets.remove(&oldest_seqnum); + } + } + + /// Returns the effective received bitrate during the last PACKETS_RECEIVED_WINDOW + fn effective_bitrate(&self) -> Bitrate { + if self.last_received_packets.is_empty() { + return 0; + } + + let duration = self + .last_received_packets + .iter() + .next_back() + .unwrap() + .1 + .arrival + - self.last_received_packets.iter().next().unwrap().1.arrival; + let bits = self + .last_received_packets + .iter() + .map(|(_seqnum, p)| p.size as f64) + .sum::<f64>() + * 8.; + + (bits + / (duration.num_nanoseconds().unwrap() as f64 + / gst::ClockTime::SECOND.nseconds() as f64)) as Bitrate + } + + fn oldest_packet_in_window_ts(&self) -> Duration { + self.last_received_packets.iter().next().unwrap().1.arrival + } + + fn update_rtts(&mut self, packets: &Vec<Packet>) { + let mut rtt = Duration::nanoseconds(i64::MAX); + let now = ts2dur(self.clock.time().unwrap()); + for packet in packets { + rtt = (now - packet.departure).min(rtt); + } + + self.rtts.push_back(rtt); + if self.rtts.len() > ROUND_TRIP_TIME_WINDOW_SIZE { + self.rtts.pop_front(); + } + } + + fn rtt(&self) -> Duration { + Duration::nanoseconds( + (self + .rtts + .iter() + .map(|d| d.num_nanoseconds().unwrap() as f64) + .sum::<f64>() + / self.rtts.len() as f64) as i64, + ) + } + + fn update(&mut self, packets: &mut Vec<Packet>) { + self.update_rtts(packets); + let mut lost_packets = 0.; + let n_packets = packets.len(); + for pkt in packets { + // We know feedbacks packets will arrive "soon" after the packets they are reported for or considered + // lost so we can make the assumption that + let mut seqnum = pkt.seqnum + (self.twcc_extended_seqnum & !(0xffff as u64)); + + if seqnum < self.twcc_extended_seqnum { + let diff = self.twcc_extended_seqnum.overflowing_sub(seqnum).0; + + if diff > i16::MAX as u64 { + seqnum += 1 << 16; + } + } else { + let diff = seqnum.overflowing_sub(self.twcc_extended_seqnum).0; + + if diff > i16::MAX as u64 { + if seqnum < 1 << 16 { + eprintln!("Cannot unwrap, any wrapping took place yet. Returning 0 without updating extended timestamp."); + } else { + seqnum -= 1 << 16; + } + } + } + + self.twcc_extended_seqnum = u64::max(seqnum, self.twcc_extended_seqnum); + + pkt.seqnum = seqnum; + + if pkt.arrival.is_zero() { + lost_packets += 1.; + continue; + } + + self.update_last_received_packets(*pkt); + + if self.group.arrival.is_none() { + self.group.add(*pkt); + + continue; + } + + if pkt.arrival < self.group.arrival.unwrap() { + // ignore out of order arrivals + continue; + } + + if pkt.departure >= self.group.departure { + if self.group.inter_departure_time_pkt(pkt) < *BURST_TIME { + self.group.add(*pkt); + continue; + } + + // 5.2 Pre-filtering + // + // A Packet which has an inter-arrival time less than burst_time and + // an inter-group delay variation d(i) less than 0 is considered + // being part of the current group of packets. + if self.group.inter_arrival_time_pkt(pkt) < *BURST_TIME + && self.group.inter_delay_variation_pkt(pkt) < Duration::zero() + { + self.group.add(*pkt); + continue; + } + + let group = mem::take(&mut self.group); + gst::trace!( + CAT, + "Packet group done: {:?}", + gst::ClockTime::from_nseconds(group.departure.num_nanoseconds().unwrap() as u64) + ); + if let Some(prev_group) = mem::replace(&mut self.prev_group, Some(group.clone())) { + // 5.3 Arrival-time filter + self.kalman_estimate(&prev_group, &group); + // 5.4 Over-use detector + self.overuse_filter(); + } + } else { + gst::debug!( + CAT, + "Ignoring packet departed at {:?} as we got feedback too late", + gst::ClockTime::from_nseconds(pkt.departure.num_nanoseconds().unwrap() as u64) + ); + } + } + + self.compute_loss_average(lost_packets as f64 / n_packets as f64); + } + + fn compute_loss_average(&mut self, loss_fraction: f64) { + let now = time::Instant::now(); + + if let Some(ref last_update) = self.last_loss_update { + self.loss_average = loss_fraction + + (-Duration::from_std(now - *last_update) + .unwrap() + .num_milliseconds() as f64) + .exp() + * (self.loss_average - loss_fraction); + } + + self.last_loss_update = Some(now); + } + + fn kalman_estimate(&mut self, prev_group: &PacketGroup, group: &PacketGroup) { + self.measure = group.inter_delay_variation(prev_group); + + let z = self.measure - self.estimate; + let zms = z.num_microseconds().unwrap() as f64 / 1000.0; + + // This doesn't exactly follows the spec as we should compute and + // use f_max here, no implementation we have found actually uses it. + let alpha = ONE_MINUS_CHI.powf(30.0 / (1000. * 5. * 1_000_000.)); + let root = self.measurement_uncertainty.sqrt(); + let root3 = 3. * root; + + if zms > root3 { + self.measurement_uncertainty = + (alpha * self.measurement_uncertainty + (1. - alpha) * root3.powf(2.)).max(1.); + } else { + self.measurement_uncertainty = + (alpha * self.measurement_uncertainty + (1. - alpha) * zms.powf(2.)).max(1.); + } + + let estimate_uncertainty = self.estimate_error + Q; + self.gain = estimate_uncertainty / (estimate_uncertainty + self.measurement_uncertainty); + self.estimate = + self.estimate + Duration::nanoseconds((self.gain * zms * 1_000_000.) as i64); + self.estimate_error = (1. - self.gain) * estimate_uncertainty; + } + + fn compare_threshold(&mut self) -> (NetworkUsage, Duration) { + // FIXME: It is unclear where that factor is coming from but all + // implementations we found have it (libwebrtc, pion, jitsi...), and the + // algorithm does not work without it. + const MAX_DELTAS: i64 = 60; + + self.num_deltas += 1; + if self.num_deltas < 2 { + return (NetworkUsage::Normal, self.estimate); + } + + let t = Duration::nanoseconds( + self.estimate.num_nanoseconds().unwrap() * i64::min(self.num_deltas, MAX_DELTAS), + ); + let usage = if t > self.threshold { + NetworkUsage::Over + } else if t.num_nanoseconds().unwrap() < -self.threshold.num_nanoseconds().unwrap() { + NetworkUsage::Under + } else { + NetworkUsage::Normal + }; + + self.update_threshold(&t); + + (usage, t) + } + + fn update_threshold(&mut self, estimate: &Duration) { + const K_U: f64 = 0.01; // Table1. Coefficient for the adaptive threshold + const K_D: f64 = 0.00018; // Table1. Coefficient for the adaptive threshold + const MAX_TIME_DELTA: time::Duration = time::Duration::from_millis(100); + + let now = time::Instant::now(); + if self.last_threshold_update.is_none() { + self.last_threshold_update = Some(now); + } + + let abs_estimate = Duration::nanoseconds(estimate.num_nanoseconds().unwrap().abs()); + if abs_estimate > self.threshold + *MAX_M_MINUS_DEL_VAR_TH { + self.last_threshold_update = Some(now); + return; + } + + let k = if abs_estimate < self.threshold { + K_D + } else { + K_U + }; + let time_delta = + Duration::from_std((now - self.last_threshold_update.unwrap()).min(MAX_TIME_DELTA)) + .unwrap(); + let d = abs_estimate - self.threshold; + let add = k * d.num_milliseconds() as f64 * time_delta.num_milliseconds() as f64; + + self.threshold = self.threshold + Duration::nanoseconds((add * 100. * 1_000.) as i64); + self.threshold = self.threshold.clamp(*MIN_THRESHOLD, *MAX_THRESHOLD); + self.last_threshold_update = Some(now); + } + + fn overuse_filter(&mut self) { + let (th_usage, estimate) = self.compare_threshold(); + + let now = time::Instant::now(); + let delta = Duration::from_std(now - self.last_use_detector_update).unwrap(); + self.last_use_detector_update = now; + gst::log!( + CAT, + "{:?} - self.estimate {} - estimate: {} - th: {}", + th_usage, + pdur(&self.estimate), + pdur(&estimate), + pdur(&self.threshold) + ); + match th_usage { + NetworkUsage::Over => { + self.increasing_duration = self.increasing_duration + delta; + self.increasing_counter += 1; + + if self.increasing_duration > *OVERUSE_TIME_TH + && self.increasing_counter > 1 + && estimate > self.last_overuse_estimate + { + self.usage = NetworkUsage::Over; + } + } + NetworkUsage::Under | NetworkUsage::Normal => { + self.increasing_duration = Duration::zero(); + self.increasing_counter = 0; + + self.usage = th_usage; + } + } + self.last_overuse_estimate = estimate; + } +} + +#[derive(Default, Debug)] +struct ExponentialMovingAverage { + average: Option<f64>, + variance: f64, + standard_dev: f64, +} + +impl ExponentialMovingAverage { + fn update<T: Into<f64>>(&mut self, value: T) { + if let Some(avg) = self.average { + let avg_diff = value.into() - avg; + + self.variance = (1. - MOVING_AVERAGE_SMOOTHING_FACTOR) + * (self.variance + MOVING_AVERAGE_SMOOTHING_FACTOR * avg_diff * avg_diff); + self.standard_dev = self.variance.sqrt(); + + self.average = Some(avg + (MOVING_AVERAGE_SMOOTHING_FACTOR * avg_diff)); + } else { + self.average = Some(value.into()); + } + } + + fn estimate_is_close(&self, value: Bitrate) -> bool { + self.average.map_or(false, |avg| { + ((avg - STANDARD_DEVIATION_CLOSE_NUM * self.standard_dev) + ..(avg + STANDARD_DEVIATION_CLOSE_NUM * self.standard_dev)) + .contains(&(value as f64)) + }) + } +} + +struct State { + /// Note: The target bitrate applied is the min of + /// target_bitrate_on_delay and target_bitrate_on_loss + estimated_bitrate: Bitrate, + + /// Bitrate target based on delay factor for all video streams. + /// Hasn't been tested with multiple video streams, but + /// current design is simply to divide bitrate equally. + target_bitrate_on_delay: Bitrate, + + /// Used in additive mode to track last control time, influences + /// calculation of added value according to gcc section 5.5 + last_increase_on_delay: Option<time::Instant>, + last_decrease_on_delay: time::Instant, + + /// Bitrate target based on loss for all video streams. + target_bitrate_on_loss: Bitrate, + + last_increase_on_loss: time::Instant, + last_decrease_on_loss: time::Instant, + + /// Exponential moving average, updated when bitrate is + /// decreased + ema: ExponentialMovingAverage, + + last_control_op: BandwidthEstimationOp, + + min_bitrate: Bitrate, + max_bitrate: Bitrate, + + detector: Detector, + + clock_entry: Option<gst::SingleShotClockId>, + + // Implemented like a leaky bucket + buffers: VecDeque<gst::Buffer>, + // Number of bits remaining from previous burst + budget_offset: i64, + + flow_return: Result<gst::FlowSuccess, gst::FlowError>, + last_push: time::Instant, +} + +impl Default for State { + fn default() -> Self { + Self { + target_bitrate_on_delay: DEFAULT_ESTIMATED_BITRATE, + target_bitrate_on_loss: DEFAULT_ESTIMATED_BITRATE, + last_increase_on_loss: time::Instant::now(), + last_decrease_on_loss: time::Instant::now(), + ema: Default::default(), + last_increase_on_delay: None, + last_decrease_on_delay: time::Instant::now(), + min_bitrate: DEFAULT_MIN_BITRATE, + max_bitrate: DEFAULT_MAX_BITRATE, + detector: Detector::new(), + buffers: Default::default(), + estimated_bitrate: DEFAULT_ESTIMATED_BITRATE, + last_control_op: BandwidthEstimationOp::Increase("Initial increase".into()), + flow_return: Err(gst::FlowError::Flushing), + clock_entry: None, + last_push: time::Instant::now(), + budget_offset: 0, + } + } +} + +impl State { + // 4. sending engine implementing a "leaky bucket" + fn create_buffer_list(&mut self, bwe: &super::BandwidthEstimator) -> gst::BufferList { + let now = time::Instant::now(); + let elapsed = Duration::from_std(now - self.last_push).unwrap(); + let mut budget = (elapsed.num_nanoseconds().unwrap()) + .mul_div_round( + self.estimated_bitrate as i64, + gst::ClockTime::SECOND.nseconds() as i64, + ) + .unwrap() + + self.budget_offset; + let total_budget = budget; + let mut remaining = self.buffers.iter().map(|b| b.size() as f64).sum::<f64>() * 8.; + let total_size = remaining; + + let mut list = gst::BufferList::new(); + let mutlist = list.get_mut().unwrap(); + + // Leak the bucket so it can hold at most 30ms of data + let maximum_remaining_bits = 30. * self.estimated_bitrate as f64 / 1000.; + let mut leaked = false; + while (budget > 0 || remaining > maximum_remaining_bits) && !self.buffers.is_empty() { + let buf = self.buffers.pop_back().unwrap(); + let n_bits = buf.size() * 8; + + leaked = budget <= 0 && remaining > maximum_remaining_bits; + mutlist.add(buf); + budget -= n_bits as i64; + remaining -= n_bits as f64; + } + + gst::trace!( + CAT, + obj: bwe, + "{} bitrate: {}ps budget: {}/{} sending: {} Remaining: {}/{}", + pdur(&elapsed), + human_kbits(self.estimated_bitrate), + human_kbits(budget as f64), + human_kbits(total_budget as f64), + human_kbits(list.calculate_size() as f64 * 8.), + human_kbits(remaining), + human_kbits(total_size) + ); + + self.last_push = now; + self.budget_offset = if !leaked { budget } else { 0 }; + + list + } + + fn compute_increased_rate(&mut self, bwe: &super::BandwidthEstimator) -> Option<Bitrate> { + let now = time::Instant::now(); + let target_bitrate = self.target_bitrate_on_delay as f64; + let effective_bitrate = self.detector.effective_bitrate(); + let time_since_last_update_ms = match self.last_increase_on_delay { + None => 0., + Some(prev) => { + if now - prev < *DELAY_UPDATE_INTERVAL { + return None; + } + + (now - prev).as_millis() as f64 + } + }; + + if effective_bitrate as f64 - target_bitrate as f64 > 5. * target_bitrate / 100. { + gst::info!( + CAT, + "Effective rate {} >> target bitrate {} - we should avoid that \ + as much as possible fine tuning the encoder", + human_kbits(effective_bitrate), + human_kbits(target_bitrate) + ); + } + + self.last_increase_on_delay = Some(now); + if self.ema.estimate_is_close(effective_bitrate) { + let bits_per_frame = target_bitrate / 30.; + let packets_per_frame = f64::ceil(bits_per_frame / (1200. * 8.)); + let avg_packet_size_bits = bits_per_frame / packets_per_frame; + + let rtt_ms = self.detector.rtt().num_milliseconds() as f64; + let response_time_ms = 100. + rtt_ms; + let alpha = 0.5 * f64::min(time_since_last_update_ms / response_time_ms, 1.0); + let threshold_on_effective_bitrate = 1.5 * effective_bitrate as f64; + let increase = f64::max( + 1000.0f64, + f64::min( + alpha * avg_packet_size_bits, + // Stuffing should ensure that the effective bitrate is not + // < target bitrate, still, make sure to always increase + // the bitrate by a minimum amount of 160.bits + f64::max( + threshold_on_effective_bitrate - self.target_bitrate_on_delay as f64, + 160., + ), + ), + ); + + /* Additive increase */ + self.last_control_op = + BandwidthEstimationOp::Increase(format!("Additive ({})", human_kbits(increase))); + Some((self.target_bitrate_on_delay as f64 + increase) as Bitrate) + } else { + let eta = 1.08_f64.powf(f64::min(time_since_last_update_ms / 1000., 1.0)); + let rate = eta * self.target_bitrate_on_delay as f64; + + self.ema = Default::default(); + + assert!( + rate >= self.target_bitrate_on_delay as f64, + "Increase: {} - {}", + rate, + eta + ); + + // Maximum increase to 1.5 * received rate + let received_max = 1.5 * effective_bitrate as f64; + + if rate > received_max && received_max > self.target_bitrate_on_delay as f64 { + gst::log!( + CAT, + obj: bwe, + "Increasing == received_max rate: {}ps", + human_kbits(received_max) + ); + + self.last_control_op = BandwidthEstimationOp::Increase(format!( + "Using 1.5*effective_rate({})", + human_kbits(effective_bitrate) + )); + Some(received_max as Bitrate) + } else if rate < self.target_bitrate_on_delay as f64 { + gst::log!( + CAT, + obj: bwe, + "Rate < target, returning {}ps", + human_kbits(self.target_bitrate_on_delay) + ); + + None + } else { + gst::log!( + CAT, + obj: bwe, + "Increase mult {eta}x{}ps={}ps", + human_kbits(self.target_bitrate_on_delay), + human_kbits(rate) + ); + + self.last_control_op = + BandwidthEstimationOp::Increase(format!("Multiplicative x{eta}")); + Some(rate as Bitrate) + } + } + } + + fn set_bitrate( + &mut self, + bwe: &super::BandwidthEstimator, + bitrate: Bitrate, + controller_type: ControllerType, + ) -> bool { + let prev_bitrate = Bitrate::min(self.target_bitrate_on_delay, self.target_bitrate_on_loss); + + match controller_type { + ControllerType::Loss => { + self.target_bitrate_on_loss = bitrate.clamp(self.min_bitrate, self.max_bitrate) + } + + ControllerType::Delay => { + self.target_bitrate_on_delay = bitrate.clamp(self.min_bitrate, self.max_bitrate) + } + } + + let target_bitrate = + Bitrate::min(self.target_bitrate_on_delay, self.target_bitrate_on_loss) + .clamp(self.min_bitrate, self.max_bitrate); + + if target_bitrate == prev_bitrate { + return false; + } + + gst::info!( + CAT, + obj: bwe, + "{controller_type:?}: {}ps => {}ps ({:?}) - effective bitrate: {}", + human_kbits(prev_bitrate), + human_kbits(target_bitrate), + self.last_control_op, + human_kbits(self.detector.effective_bitrate()), + ); + + self.estimated_bitrate = target_bitrate; + + true + } + + fn loss_control(&mut self, bwe: &super::BandwidthEstimator) -> bool { + let loss_ratio = self.detector.loss_ratio(); + let now = time::Instant::now(); + + if loss_ratio > LOSS_DECREASE_THRESHOLD + && (now - self.last_decrease_on_loss) > *LOSS_UPDATE_INTERVAL + { + let factor = 1. - (0.5 * loss_ratio); + + self.last_control_op = + BandwidthEstimationOp::Decrease(format!("High loss detected ({loss_ratio:2}")); + self.last_decrease_on_loss = now; + + self.set_bitrate( + bwe, + (self.target_bitrate_on_loss as f64 * factor) as Bitrate, + ControllerType::Loss, + ) + } else if loss_ratio < LOSS_INCREASE_THRESHOLD + && (now - self.last_increase_on_loss) > *LOSS_UPDATE_INTERVAL + { + self.last_control_op = BandwidthEstimationOp::Increase("Low loss".into()); + self.last_increase_on_loss = now; + + self.set_bitrate( + bwe, + (self.target_bitrate_on_loss as f64 * LOSS_INCREASE_FACTOR) as Bitrate, + ControllerType::Loss, + ) + } else { + false + } + } + + fn delay_control(&mut self, bwe: &super::BandwidthEstimator) -> bool { + match self.detector.usage { + NetworkUsage::Normal => match self.last_control_op { + BandwidthEstimationOp::Increase(..) | BandwidthEstimationOp::Hold => { + if let Some(bitrate) = self.compute_increased_rate(bwe) { + return self.set_bitrate(bwe, bitrate, ControllerType::Delay); + } + } + _ => (), + }, + NetworkUsage::Over => { + let now = time::Instant::now(); + if now - self.last_decrease_on_delay > *DELAY_UPDATE_INTERVAL { + let effective_bitrate = self.detector.effective_bitrate(); + let target = + (self.estimated_bitrate as f64 * 0.95).min(BETA * effective_bitrate as f64); + self.last_control_op = BandwidthEstimationOp::Decrease(format!( + "Over use detected {:#?}", + self.detector + )); + self.ema.update(effective_bitrate); + self.last_decrease_on_delay = now; + + return self.set_bitrate(bwe, target as Bitrate, ControllerType::Delay); + } + } + NetworkUsage::Under => { + if let BandwidthEstimationOp::Increase(..) = self.last_control_op { + if let Some(bitrate) = self.compute_increased_rate(bwe) { + return self.set_bitrate(bwe, bitrate, ControllerType::Delay); + } + } + } + } + + self.last_control_op = BandwidthEstimationOp::Hold; + + false + } +} + +pub struct BandwidthEstimator { + state: Mutex<State>, + + srcpad: gst::Pad, + sinkpad: gst::Pad, +} + +impl BandwidthEstimator { + fn push_list(&self, list: gst::BufferList) -> Result<gst::FlowSuccess, gst::FlowError> { + let res = self.srcpad.push_list(list); + + self.state.lock().unwrap().flow_return = res; + + res + } + + fn start_task(&self, bwe: &super::BandwidthEstimator) -> Result<(), glib::BoolError> { + let weak_bwe = bwe.downgrade(); + let weak_pad = self.srcpad.downgrade(); + let clock = gst::SystemClock::obtain(); + + bwe.imp().state.lock().unwrap().clock_entry = + Some(clock.new_single_shot_id(clock.time().unwrap() + dur2ts(*BURST_TIME))); + + self.srcpad.start_task(move || { + let pause = || { + if let Some(pad) = weak_pad.upgrade() { + let _ = pad.pause_task(); + } + }; + let bwe = weak_bwe + .upgrade() + .expect("bwe destroyed while its srcpad task is still running?"); + + let lock_state = || bwe.imp().state.lock().unwrap(); + + let clock_entry = match lock_state().clock_entry.take() { + Some(id) => id, + _ => { + gst::info!(CAT, "Pausing task as our clock entry is not set anymore"); + return pause(); + } + }; + + if let (Err(err), _) = clock_entry.wait() { + match err { + gst::ClockError::Early => (), + _ => { + gst::error!(CAT, "Got error {err:?} on the clock, pausing task"); + + lock_state().flow_return = Err(gst::FlowError::Flushing); + + return pause(); + } + } + } + let list = { + let mut state = lock_state(); + clock + .single_shot_id_reinit( + &clock_entry, + clock.time().unwrap() + dur2ts(*BURST_TIME), + ) + .unwrap(); + state.clock_entry = Some(clock_entry); + state.create_buffer_list(&bwe) + }; + + if !list.is_empty() { + if let Err(err) = bwe.imp().push_list(list) { + gst::error!(CAT, obj: &bwe, "pause task, reason: {err:?}"); + return pause(); + } + } + })?; + + Ok(()) + } + + fn src_activatemode( + &self, + _pad: &gst::Pad, + bwe: &super::BandwidthEstimator, + mode: gst::PadMode, + active: bool, + ) -> Result<(), gst::LoggableError> { + if let gst::PadMode::Push = mode { + if active { + self.state.lock().unwrap().flow_return = Ok(gst::FlowSuccess::Ok); + self.start_task(bwe)?; + } else { + let mut state = self.state.lock().unwrap(); + state.flow_return = Err(gst::FlowError::Flushing); + drop(state); + + self.srcpad.stop_task()?; + } + + Ok(()) + } else { + Err(gst::LoggableError::new( + *CAT, + glib::bool_error!("Unsupported pad mode {mode:?}"), + )) + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for BandwidthEstimator { + const NAME: &'static str = "GstRTPGCCBwE"; + type Type = super::BandwidthEstimator; + type ParentType = gst::Element; + + fn with_class(klass: &Self::Class) -> Self { + let templ = klass.pad_template("sink").unwrap(); + let sinkpad = gst::Pad::builder_with_template(&templ, Some("sink")) + .chain_function(|_pad, parent, mut buffer| { + BandwidthEstimator::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |this| { + let mut state = this.state.lock().unwrap(); + let mutbuf = buffer.make_mut(); + mutbuf.set_pts(None); + mutbuf.set_dts(None); + state.buffers.push_front(buffer); + + state.flow_return + }, + ) + }) + .flags(gst::PadFlags::PROXY_CAPS | gst::PadFlags::PROXY_ALLOCATION) + .build(); + + let templ = klass.pad_template("src").unwrap(); + let srcpad = gst::Pad::builder_with_template(&templ, Some("src")) + .event_function(|pad, parent, event| { + BandwidthEstimator::catch_panic_pad_function( + parent, + || false, + |this| { + let bwe = this.instance(); + + if let Some(structure) = event.structure() { + if structure.name() == "RTPTWCCPackets" { + let varray = structure.get::<glib::ValueArray>("packets").unwrap(); + let mut packets = varray + .iter() + .filter_map(|s| { + Packet::from_structure(&s.get::<gst::Structure>().unwrap()) + }) + .collect::<Vec<Packet>>(); + + let bitrate_changed = { + let mut state = this.state.lock().unwrap(); + + state.detector.update(&mut packets); + if !state.delay_control(&bwe) { + state.loss_control(&bwe) + } else { + true + } + }; + + if bitrate_changed { + bwe.notify("estimated-bitrate") + } + } + } + + gst::Pad::event_default(pad, parent, event) + }, + ) + }) + .activatemode_function(|pad, parent, mode, active| { + BandwidthEstimator::catch_panic_pad_function( + parent, + || { + Err(gst::loggable_error!( + CAT, + "Panic activating src pad with mode" + )) + }, + |this| this.src_activatemode(pad, &this.instance(), mode, active), + ) + }) + .flags(gst::PadFlags::PROXY_CAPS | gst::PadFlags::PROXY_ALLOCATION) + .build(); + + Self { + state: Default::default(), + srcpad, + sinkpad, + } + } +} + +impl ObjectImpl for BandwidthEstimator { + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.instance(); + obj.add_pad(&self.sinkpad).unwrap(); + obj.add_pad(&self.srcpad).unwrap(); + } + + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { + vec![ + /* + * gcc:estimated-bitrate: + * + * Currently computed network bitrate, should be used + * to set encoders bitrate. + */ + glib::ParamSpecUInt::new( + "estimated-bitrate", + "Estimated Bitrate", + "Currently estimated bitrate. Can be set before starting + the element to configure the starting bitrate, in which case the + encoder should also use it as target bitrate", + 1, + u32::MAX as u32, + DEFAULT_MIN_BITRATE as u32, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecUInt::new( + "min-bitrate", + "Minimal Bitrate", + "Minimal bitrate to use (in bit/sec) when computing it through the bandwidth estimation algorithm", + 1, + u32::MAX as u32, + DEFAULT_MIN_BITRATE, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecUInt::new( + "max-bitrate", + "Maximal Bitrate", + "Maximal bitrate to use (in bit/sec) when computing it through the bandwidth estimation algorithm", + 1, + u32::MAX as u32, + DEFAULT_MAX_BITRATE, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property( + &self, + _id: usize, + value: &glib::Value, + pspec: &glib::ParamSpec, + ) { + match pspec.name() { + "min-bitrate" => { + let mut state = self.state.lock().unwrap(); + state.min_bitrate = value.get::<u32>().expect("type checked upstream"); + } + "max-bitrate" => { + let mut state = self.state.lock().unwrap(); + state.max_bitrate = value.get::<u32>().expect("type checked upstream"); + } + "estimated-bitrate" => { + let mut state = self.state.lock().unwrap(); + let bitrate = value.get::<u32>().expect("type checked upstream"); + state.target_bitrate_on_delay = bitrate; + state.target_bitrate_on_loss = bitrate; + state.estimated_bitrate = bitrate; + } + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "min-bitrate" => { + let state = self.state.lock().unwrap(); + state.min_bitrate.to_value() + } + "max-bitrate" => { + let state = self.state.lock().unwrap(); + state.max_bitrate.to_value() + } + "estimated-bitrate" => { + let state = self.state.lock().unwrap(); + state.estimated_bitrate.to_value() + } + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for BandwidthEstimator {} + +impl ElementImpl for BandwidthEstimator { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "Google Congestion Control bandwidth estimator", + "Network/WebRTC/RTP/Filter", + "Estimates current network bandwidth using the Google Congestion Control algorithm \ + notifying about it through the 'bitrate' property", + "Thibault Saunier <tsaunier@igalia.com>", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| { + let caps = gst::Caps::builder_full() + .structure(gst::Structure::builder("application/x-rtp").build()) + .build(); + + let sinkpad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + let srcpad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![sinkpad_template, srcpad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} diff --git a/net/webrtc/plugins/src/gcc/mod.rs b/net/webrtc/plugins/src/gcc/mod.rs new file mode 100644 index 00000000..5bbc07db --- /dev/null +++ b/net/webrtc/plugins/src/gcc/mod.rs @@ -0,0 +1,16 @@ +use gst::glib; +use gst::prelude::*; +mod imp; + +glib::wrapper! { + pub struct BandwidthEstimator(ObjectSubclass<imp::BandwidthEstimator>) @extends gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "rtpgccbwe", + gst::Rank::None, + BandwidthEstimator::static_type(), + ) +} diff --git a/net/webrtc/plugins/src/lib.rs b/net/webrtc/plugins/src/lib.rs new file mode 100644 index 00000000..81a65cea --- /dev/null +++ b/net/webrtc/plugins/src/lib.rs @@ -0,0 +1,24 @@ +use gst::glib; + +pub mod gcc; +mod signaller; +pub mod webrtcsink; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + webrtcsink::register(plugin)?; + gcc::register(plugin)?; + + Ok(()) +} + +gst::plugin_define!( + webrtcsink, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MPL-2.0", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/net/webrtc/plugins/src/signaller/imp.rs b/net/webrtc/plugins/src/signaller/imp.rs new file mode 100644 index 00000000..74415c4b --- /dev/null +++ b/net/webrtc/plugins/src/signaller/imp.rs @@ -0,0 +1,478 @@ +use crate::webrtcsink::WebRTCSink; +use anyhow::{anyhow, Error}; +use async_std::task; +use async_tungstenite::tungstenite::Message as WsMessage; +use futures::channel::mpsc; +use futures::prelude::*; +use gst::glib::prelude::*; +use gst::glib::{self, Type}; +use gst::prelude::*; +use gst::subclass::prelude::*; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; +use webrtcsink_protocol as p; + +static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { + gst::DebugCategory::new( + "webrtcsink-signaller", + gst::DebugColorFlags::empty(), + Some("WebRTC sink signaller"), + ) +}); + +#[derive(Default)] +struct State { + /// Sender for the websocket messages + websocket_sender: Option<mpsc::Sender<p::IncomingMessage>>, + send_task_handle: Option<task::JoinHandle<Result<(), Error>>>, + receive_task_handle: Option<task::JoinHandle<()>>, +} + +#[derive(Clone)] +struct Settings { + address: Option<String>, + cafile: Option<PathBuf>, +} + +impl Default for Settings { + fn default() -> Self { + Self { + address: Some("ws://127.0.0.1:8443".to_string()), + cafile: None, + } + } +} + +#[derive(Default)] +pub struct Signaller { + state: Mutex<State>, + settings: Mutex<Settings>, +} + +impl Signaller { + async fn connect(&self, element: &WebRTCSink) -> Result<(), Error> { + let settings = self.settings.lock().unwrap().clone(); + + let connector = if let Some(path) = settings.cafile { + let cert = async_std::fs::read_to_string(&path).await?; + let cert = async_native_tls::Certificate::from_pem(cert.as_bytes())?; + let connector = async_native_tls::TlsConnector::new(); + Some(connector.add_root_certificate(cert)) + } else { + None + }; + + let (ws, _) = async_tungstenite::async_std::connect_async_with_tls_connector( + settings.address.unwrap(), + connector, + ) + .await?; + + gst::info!(CAT, obj: element, "connected"); + + // Channel for asynchronously sending out websocket message + let (mut ws_sink, mut ws_stream) = ws.split(); + + // 1000 is completely arbitrary, we simply don't want infinite piling + // up of messages as with unbounded + let (mut websocket_sender, mut websocket_receiver) = + mpsc::channel::<p::IncomingMessage>(1000); + let element_clone = element.downgrade(); + let send_task_handle = task::spawn(async move { + while let Some(msg) = websocket_receiver.next().await { + if let Some(element) = element_clone.upgrade() { + gst::trace!(CAT, obj: &element, "Sending websocket message {:?}", msg); + } + ws_sink + .send(WsMessage::Text(serde_json::to_string(&msg).unwrap())) + .await?; + } + + if let Some(element) = element_clone.upgrade() { + gst::info!(CAT, obj: &element, "Done sending"); + } + + ws_sink.send(WsMessage::Close(None)).await?; + ws_sink.close().await?; + + Ok::<(), Error>(()) + }); + + let meta = if let Some(meta) = element.property::<Option<gst::Structure>>("meta") { + serialize_value(&meta.to_value()) + } else { + None + }; + websocket_sender + .send(p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta, + peer_id: None, + })) + .await?; + + let element_clone = element.downgrade(); + let receive_task_handle = task::spawn(async move { + while let Some(msg) = async_std::stream::StreamExt::next(&mut ws_stream).await { + if let Some(element) = element_clone.upgrade() { + match msg { + Ok(WsMessage::Text(msg)) => { + gst::trace!(CAT, obj: &element, "Received message {}", msg); + + if let Ok(msg) = serde_json::from_str::<p::OutgoingMessage>(&msg) { + match msg { + p::OutgoingMessage::Welcome { peer_id } => { + gst::info!( + CAT, + obj: &element, + "We are registered with the server, our peer id is {}", + peer_id + ); + } + p::OutgoingMessage::StartSession { + session_id, + peer_id, + } => { + if let Err(err) = + element.start_session(&session_id, &peer_id) + { + gst::warning!(CAT, obj: &element, "{}", err); + } + } + p::OutgoingMessage::EndSession(session_info) => { + if let Err(err) = + element.end_session(&session_info.session_id) + { + gst::warning!(CAT, obj: &element, "{}", err); + } + } + p::OutgoingMessage::Peer(p::PeerMessage { + session_id, + peer_message, + }) => match peer_message { + p::PeerMessageInner::Sdp(p::SdpMessage::Answer { sdp }) => { + if let Err(err) = element.handle_sdp( + &session_id, + &gst_webrtc::WebRTCSessionDescription::new( + gst_webrtc::WebRTCSDPType::Answer, + gst_sdp::SDPMessage::parse_buffer( + sdp.as_bytes(), + ) + .unwrap(), + ), + ) { + gst::warning!(CAT, obj: &element, "{}", err); + } + } + p::PeerMessageInner::Sdp(p::SdpMessage::Offer { + .. + }) => { + gst::warning!( + CAT, + obj: &element, + "Ignoring offer from peer" + ); + } + p::PeerMessageInner::Ice { + candidate, + sdp_m_line_index, + } => { + if let Err(err) = element.handle_ice( + &session_id, + Some(sdp_m_line_index), + None, + &candidate, + ) { + gst::warning!(CAT, obj: &element, "{}", err); + } + } + }, + _ => { + gst::warning!( + CAT, + obj: &element, + "Ignoring unsupported message {:?}", + msg + ); + } + } + } else { + gst::error!( + CAT, + obj: &element, + "Unknown message from server: {}", + msg + ); + element.handle_signalling_error( + anyhow!("Unknown message from server: {}", msg).into(), + ); + } + } + Ok(WsMessage::Close(reason)) => { + gst::info!( + CAT, + obj: &element, + "websocket connection closed: {:?}", + reason + ); + break; + } + Ok(_) => (), + Err(err) => { + element.handle_signalling_error( + anyhow!("Error receiving: {}", err).into(), + ); + break; + } + } + } else { + break; + } + } + + if let Some(element) = element_clone.upgrade() { + gst::info!(CAT, obj: &element, "Stopped websocket receiving"); + } + }); + + let mut state = self.state.lock().unwrap(); + state.websocket_sender = Some(websocket_sender); + state.send_task_handle = Some(send_task_handle); + state.receive_task_handle = Some(receive_task_handle); + + Ok(()) + } + + pub fn start(&self, element: &WebRTCSink) { + let this = self.instance().clone(); + let element_clone = element.clone(); + task::spawn(async move { + let this = Self::from_instance(&this); + if let Err(err) = this.connect(&element_clone).await { + element_clone.handle_signalling_error(err.into()); + } + }); + } + + pub fn handle_sdp( + &self, + element: &WebRTCSink, + session_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) { + let state = self.state.lock().unwrap(); + + let msg = p::IncomingMessage::Peer(p::PeerMessage { + session_id: session_id.to_string(), + peer_message: p::PeerMessageInner::Sdp(p::SdpMessage::Offer { + sdp: sdp.sdp().as_text().unwrap(), + }), + }); + + if let Some(mut sender) = state.websocket_sender.clone() { + let element = element.downgrade(); + task::spawn(async move { + if let Err(err) = sender.send(msg).await { + if let Some(element) = element.upgrade() { + element.handle_signalling_error(anyhow!("Error: {}", err).into()); + } + } + }); + } + } + + pub fn handle_ice( + &self, + element: &WebRTCSink, + session_id: &str, + candidate: &str, + sdp_m_line_index: Option<u32>, + _sdp_mid: Option<String>, + ) { + let state = self.state.lock().unwrap(); + + let msg = p::IncomingMessage::Peer(p::PeerMessage { + session_id: session_id.to_string(), + peer_message: p::PeerMessageInner::Ice { + candidate: candidate.to_string(), + sdp_m_line_index: sdp_m_line_index.unwrap(), + }, + }); + + if let Some(mut sender) = state.websocket_sender.clone() { + let element = element.downgrade(); + task::spawn(async move { + if let Err(err) = sender.send(msg).await { + if let Some(element) = element.upgrade() { + element.handle_signalling_error(anyhow!("Error: {}", err).into()); + } + } + }); + } + } + + pub fn stop(&self, element: &WebRTCSink) { + gst::info!(CAT, obj: element, "Stopping now"); + + let mut state = self.state.lock().unwrap(); + let send_task_handle = state.send_task_handle.take(); + let receive_task_handle = state.receive_task_handle.take(); + if let Some(mut sender) = state.websocket_sender.take() { + task::block_on(async move { + sender.close_channel(); + + if let Some(handle) = send_task_handle { + if let Err(err) = handle.await { + gst::warning!(CAT, obj: element, "Error while joining send task: {}", err); + } + } + + if let Some(handle) = receive_task_handle { + handle.await; + } + }); + } + } + + pub fn end_session(&self, element: &WebRTCSink, session_id: &str) { + gst::debug!(CAT, obj: element, "Signalling session {} ended", session_id); + + let state = self.state.lock().unwrap(); + let session_id = session_id.to_string(); + let element = element.downgrade(); + if let Some(mut sender) = state.websocket_sender.clone() { + task::spawn(async move { + if let Err(err) = sender + .send(p::IncomingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.to_string(), + })) + .await + { + if let Some(element) = element.upgrade() { + element.handle_signalling_error(anyhow!("Error: {}", err).into()); + } + } + }); + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for Signaller { + const NAME: &'static str = "RsWebRTCSinkSignaller"; + type Type = super::Signaller; + type ParentType = glib::Object; +} + +impl ObjectImpl for Signaller { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { + vec![ + glib::ParamSpecString::new( + "address", + "Address", + "Address of the signalling server", + Some("ws://127.0.0.1:8443"), + glib::ParamFlags::READWRITE, + ), + glib::ParamSpecString::new( + "cafile", + "CA file", + "Path to a Certificate file to add to the set of roots the TLS connector will trust", + None, + glib::ParamFlags::READWRITE, + ), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property( + &self, + _id: usize, + value: &glib::Value, + pspec: &glib::ParamSpec, + ) { + match pspec.name() { + "address" => { + let address: Option<_> = value.get().expect("type checked upstream"); + + if let Some(address) = address { + gst::info!(CAT, "Signaller address set to {}", address); + + let mut settings = self.settings.lock().unwrap(); + settings.address = Some(address); + } else { + gst::error!(CAT, "address can't be None"); + } + } + "cafile" => { + let value: String = value.get().unwrap(); + let mut settings = self.settings.lock().unwrap(); + settings.cafile = Some(value.into()); + } + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "address" => self.settings.lock().unwrap().address.to_value(), + "cafile" => { + let settings = self.settings.lock().unwrap(); + let cafile = settings.cafile.as_ref(); + cafile.and_then(|file| file.to_str()).to_value() + } + _ => unimplemented!(), + } + } +} + +fn serialize_value(val: &gst::glib::Value) -> Option<serde_json::Value> { + match val.type_() { + Type::STRING => Some(val.get::<String>().unwrap().into()), + Type::BOOL => Some(val.get::<bool>().unwrap().into()), + Type::I32 => Some(val.get::<i32>().unwrap().into()), + Type::U32 => Some(val.get::<u32>().unwrap().into()), + Type::I_LONG | Type::I64 => Some(val.get::<i64>().unwrap().into()), + Type::U_LONG | Type::U64 => Some(val.get::<u64>().unwrap().into()), + Type::F32 => Some(val.get::<f32>().unwrap().into()), + Type::F64 => Some(val.get::<f64>().unwrap().into()), + _ => { + if let Ok(s) = val.get::<gst::Structure>() { + serde_json::to_value( + s.iter() + .filter_map(|(name, value)| { + serialize_value(value).map(|value| (name.to_string(), value)) + }) + .collect::<HashMap<String, serde_json::Value>>(), + ) + .ok() + } else if let Ok(a) = val.get::<gst::Array>() { + serde_json::to_value( + a.iter() + .filter_map(|value| serialize_value(value)) + .collect::<Vec<serde_json::Value>>(), + ) + .ok() + } else if let Some((_klass, values)) = gst::glib::FlagsValue::from_value(val) { + Some( + values + .iter() + .map(|value| value.nick()) + .collect::<Vec<&str>>() + .join("+") + .into(), + ) + } else if let Ok(value) = val.serialize() { + Some(value.as_str().into()) + } else { + gst::warning!(CAT, "Can't convert {} to json", val.type_().name()); + None + } + } + } +} diff --git a/net/webrtc/plugins/src/signaller/mod.rs b/net/webrtc/plugins/src/signaller/mod.rs new file mode 100644 index 00000000..51374254 --- /dev/null +++ b/net/webrtc/plugins/src/signaller/mod.rs @@ -0,0 +1,62 @@ +use crate::webrtcsink::{Signallable, WebRTCSink}; +use gst::glib; +use gst::subclass::prelude::ObjectSubclassExt; +use std::error::Error; + +mod imp; + +glib::wrapper! { + pub struct Signaller(ObjectSubclass<imp::Signaller>); +} + +unsafe impl Send for Signaller {} +unsafe impl Sync for Signaller {} + +impl Signallable for Signaller { + fn start(&mut self, element: &WebRTCSink) -> Result<(), Box<dyn Error>> { + let signaller = imp::Signaller::from_instance(self); + signaller.start(element); + + Ok(()) + } + + fn handle_sdp( + &mut self, + element: &WebRTCSink, + peer_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) -> Result<(), Box<dyn Error>> { + let signaller = imp::Signaller::from_instance(self); + signaller.handle_sdp(element, peer_id, sdp); + Ok(()) + } + + fn handle_ice( + &mut self, + element: &WebRTCSink, + session_id: &str, + candidate: &str, + sdp_mline_index: Option<u32>, + sdp_mid: Option<String>, + ) -> Result<(), Box<dyn Error>> { + let signaller = imp::Signaller::from_instance(self); + signaller.handle_ice(element, session_id, candidate, sdp_mline_index, sdp_mid); + Ok(()) + } + + fn stop(&mut self, element: &WebRTCSink) { + let signaller = imp::Signaller::from_instance(self); + signaller.stop(element); + } + + fn session_ended(&mut self, element: &WebRTCSink, session_id: &str) { + let signaller = imp::Signaller::from_instance(self); + signaller.end_session(element, session_id); + } +} + +impl Default for Signaller { + fn default() -> Self { + glib::Object::new::<Self>(&[]) + } +} diff --git a/net/webrtc/plugins/src/webrtcsink/homegrown_cc.rs b/net/webrtc/plugins/src/webrtcsink/homegrown_cc.rs new file mode 100644 index 00000000..04b5258f --- /dev/null +++ b/net/webrtc/plugins/src/webrtcsink/homegrown_cc.rs @@ -0,0 +1,420 @@ +use gst::{ + glib::{self, value::FromValue}, + prelude::*, +}; +use once_cell::sync::Lazy; + +use super::imp::VideoEncoder; + +static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { + gst::DebugCategory::new( + "webrtcsink-homegrowncc", + gst::DebugColorFlags::empty(), + Some("WebRTC sink"), + ) +}); + +#[derive(Debug)] +enum IncreaseType { + /// Increase bitrate by value + Additive(f64), + /// Increase bitrate by factor + Multiplicative(f64), +} + +#[derive(Debug, Clone, Copy)] +enum ControllerType { + // Running the "delay-based controller" + Delay, + // Running the "loss based controller" + Loss, +} + +#[derive(Debug)] +enum CongestionControlOp { + /// Don't update target bitrate + Hold, + /// Decrease target bitrate + Decrease { + factor: f64, + #[allow(dead_code)] + reason: String, // for Debug + }, + /// Increase target bitrate, either additively or multiplicatively + Increase(IncreaseType), +} + +fn lookup_twcc_stats(stats: &gst::StructureRef) -> Option<gst::Structure> { + for (_, field_value) in stats { + if let Ok(s) = field_value.get::<gst::Structure>() { + if let Ok(type_) = s.get::<gst_webrtc::WebRTCStatsType>("type") { + if (type_ == gst_webrtc::WebRTCStatsType::Transport + || type_ == gst_webrtc::WebRTCStatsType::CandidatePair) + && s.has_field("gst-twcc-stats") + { + return Some(s.get::<gst::Structure>("gst-twcc-stats").unwrap()); + } + } + } + } + + None +} + +pub struct CongestionController { + /// Note: The target bitrate applied is the min of + /// target_bitrate_on_delay and target_bitrate_on_loss + /// + /// Bitrate target based on delay factor for all video streams. + /// Hasn't been tested with multiple video streams, but + /// current design is simply to divide bitrate equally. + pub target_bitrate_on_delay: i32, + + /// Bitrate target based on loss for all video streams. + pub target_bitrate_on_loss: i32, + + /// Exponential moving average, updated when bitrate is + /// decreased, discarded when increased again past last + /// congestion window. Smoothing factor hardcoded. + bitrate_ema: Option<f64>, + /// Exponentially weighted moving variance, recursively + /// updated along with bitrate_ema. sqrt'd to obtain standard + /// deviation, used to determine whether to increase bitrate + /// additively or multiplicatively + bitrate_emvar: f64, + /// Used in additive mode to track last control time, influences + /// calculation of added value according to gcc section 5.5 + last_update_time: Option<std::time::Instant>, + /// For logging purposes + peer_id: String, + + min_bitrate: u32, + max_bitrate: u32, +} + +impl CongestionController { + pub fn new(peer_id: &str, min_bitrate: u32, max_bitrate: u32) -> Self { + Self { + target_bitrate_on_delay: 0, + target_bitrate_on_loss: 0, + bitrate_ema: None, + bitrate_emvar: 0., + last_update_time: None, + peer_id: peer_id.to_string(), + min_bitrate, + max_bitrate, + } + } + + fn update_delay( + &mut self, + element: &super::WebRTCSink, + twcc_stats: &gst::StructureRef, + rtt: f64, + ) -> CongestionControlOp { + let target_bitrate = f64::min( + self.target_bitrate_on_delay as f64, + self.target_bitrate_on_loss as f64, + ); + // Unwrap, all those fields must be there or there's been an API + // break, which qualifies as programming error + let bitrate_sent = twcc_stats.get::<u32>("bitrate-sent").unwrap(); + let bitrate_recv = twcc_stats.get::<u32>("bitrate-recv").unwrap(); + let delta_of_delta = twcc_stats.get::<i64>("avg-delta-of-delta").unwrap(); + + let sent_minus_received = bitrate_sent.saturating_sub(bitrate_recv); + + let delay_factor = sent_minus_received as f64 / target_bitrate; + let last_update_time = self.last_update_time.replace(std::time::Instant::now()); + + gst::trace!( + CAT, + obj: element, + "consumer {}: considering stats {}", + self.peer_id, + twcc_stats + ); + + if delay_factor > 0.1 { + let (factor, reason) = if delay_factor < 0.64 { + (0.96, format!("low delay factor {}", delay_factor)) + } else { + ( + delay_factor.sqrt().sqrt().clamp(0.8, 0.96), + format!("High delay factor {}", delay_factor), + ) + }; + + CongestionControlOp::Decrease { factor, reason } + } else if delta_of_delta > 1_000_000 { + CongestionControlOp::Decrease { + factor: 0.97, + reason: format!("High delta: {}", delta_of_delta), + } + } else { + CongestionControlOp::Increase(if let Some(ema) = self.bitrate_ema { + let bitrate_stdev = self.bitrate_emvar.sqrt(); + + gst::trace!( + CAT, + obj: element, + "consumer {}: Old bitrate: {}, ema: {}, stddev: {}", + self.peer_id, + target_bitrate, + ema, + bitrate_stdev, + ); + + // gcc section 5.5 advises 3 standard deviations, but experiments + // have shown this to be too low, probably related to the rest of + // homegrown algorithm not implementing gcc, revisit when implementing + // the rest of the RFC + if target_bitrate < ema - 7. * bitrate_stdev { + gst::trace!( + CAT, + obj: element, + "consumer {}: below last congestion window", + self.peer_id + ); + /* Multiplicative increase */ + IncreaseType::Multiplicative(1.03) + } else if target_bitrate > ema + 7. * bitrate_stdev { + gst::trace!( + CAT, + obj: element, + "consumer {}: above last congestion window", + self.peer_id + ); + /* We have gone past our last estimated max bandwidth + * network situation may have changed, go back to + * multiplicative increase + */ + self.bitrate_ema.take(); + IncreaseType::Multiplicative(1.03) + } else { + let rtt_ms = rtt * 1000.; + let response_time_ms = 100. + rtt_ms; + let time_since_last_update_ms = match last_update_time { + None => 0., + Some(instant) => { + (self.last_update_time.unwrap() - instant).as_millis() as f64 + } + }; + // gcc section 5.5 advises 0.95 as the smoothing factor, but that + // seems intuitively much too low, granting disproportionate importance + // to the last measurement. 0.5 seems plenty enough, I don't have maths + // to back that up though :) + let alpha = 0.5 * f64::min(time_since_last_update_ms / response_time_ms, 1.0); + let bits_per_frame = target_bitrate / 30.; + let packets_per_frame = f64::ceil(bits_per_frame / (1200. * 8.)); + let avg_packet_size_bits = bits_per_frame / packets_per_frame; + + gst::trace!( + CAT, + obj: element, + "consumer {}: still in last congestion window", + self.peer_id, + ); + + /* Additive increase */ + IncreaseType::Additive(f64::max(1000., alpha * avg_packet_size_bits)) + } + } else { + /* Multiplicative increase */ + gst::trace!( + CAT, + obj: element, + "consumer {}: outside congestion window", + self.peer_id + ); + IncreaseType::Multiplicative(1.03) + }) + } + } + + fn clamp_bitrate(&mut self, bitrate: i32, n_encoders: i32, controller_type: ControllerType) { + match controller_type { + ControllerType::Loss => { + self.target_bitrate_on_loss = bitrate.clamp( + self.min_bitrate as i32 * n_encoders, + self.max_bitrate as i32 * n_encoders, + ) + } + + ControllerType::Delay => { + self.target_bitrate_on_delay = bitrate.clamp( + self.min_bitrate as i32 * n_encoders, + self.max_bitrate as i32 * n_encoders, + ) + } + } + } + + fn get_remote_inbound_stats(&self, stats: &gst::StructureRef) -> Vec<gst::Structure> { + let mut inbound_rtp_stats: Vec<gst::Structure> = Default::default(); + for (_, field_value) in stats { + if let Ok(s) = field_value.get::<gst::Structure>() { + if let Ok(type_) = s.get::<gst_webrtc::WebRTCStatsType>("type") { + if type_ == gst_webrtc::WebRTCStatsType::RemoteInboundRtp { + inbound_rtp_stats.push(s); + } + } + } + } + + inbound_rtp_stats + } + + fn lookup_rtt(&self, stats: &gst::StructureRef) -> f64 { + let inbound_rtp_stats = self.get_remote_inbound_stats(stats); + let mut rtt = 0.; + let mut n_rtts = 0u64; + for inbound_stat in &inbound_rtp_stats { + if let Err(err) = (|| -> Result<(), gst::structure::GetError<<<f64 as FromValue>::Checker as glib::value::ValueTypeChecker>::Error>> { + rtt += inbound_stat.get::<f64>("round-trip-time")?; + n_rtts += 1; + + Ok(()) + })() { + gst::debug!(CAT, "{:?}", err); + } + } + + rtt /= f64::max(1., n_rtts as f64); + + gst::log!(CAT, "Round trip time: {}", rtt); + + rtt + } + + pub fn loss_control( + &mut self, + element: &super::WebRTCSink, + stats: &gst::StructureRef, + encoders: &mut Vec<VideoEncoder>, + ) { + let loss_percentage = stats.get::<f64>("packet-loss-pct").unwrap(); + + self.apply_control_op( + element, + encoders, + if loss_percentage > 10. { + CongestionControlOp::Decrease { + factor: ((100. - (0.5 * loss_percentage)) / 100.).clamp(0.7, 0.98), + reason: format!("High loss: {}", loss_percentage), + } + } else if loss_percentage > 2. { + CongestionControlOp::Hold + } else { + CongestionControlOp::Increase(IncreaseType::Multiplicative(1.05)) + }, + ControllerType::Loss, + ); + } + + pub fn delay_control( + &mut self, + element: &super::WebRTCSink, + stats: &gst::StructureRef, + encoders: &mut Vec<VideoEncoder>, + ) { + if let Some(twcc_stats) = lookup_twcc_stats(stats) { + let op = self.update_delay(element, &twcc_stats, self.lookup_rtt(stats)); + self.apply_control_op(element, encoders, op, ControllerType::Delay); + } + } + + fn apply_control_op( + &mut self, + element: &super::WebRTCSink, + encoders: &mut Vec<VideoEncoder>, + control_op: CongestionControlOp, + controller_type: ControllerType, + ) { + gst::trace!( + CAT, + obj: element, + "consumer {}: applying congestion control operation {:?}", + self.peer_id, + control_op + ); + + let n_encoders = encoders.len() as i32; + let prev_bitrate = i32::min(self.target_bitrate_on_delay, self.target_bitrate_on_loss); + match &control_op { + CongestionControlOp::Hold => {} + CongestionControlOp::Increase(IncreaseType::Additive(value)) => { + self.clamp_bitrate( + self.target_bitrate_on_delay + *value as i32, + n_encoders, + controller_type, + ); + } + CongestionControlOp::Increase(IncreaseType::Multiplicative(factor)) => { + self.clamp_bitrate( + (self.target_bitrate_on_delay as f64 * factor) as i32, + n_encoders, + controller_type, + ); + } + CongestionControlOp::Decrease { factor, .. } => { + self.clamp_bitrate( + (self.target_bitrate_on_delay as f64 * factor) as i32, + n_encoders, + controller_type, + ); + + if let ControllerType::Delay = controller_type { + // Smoothing factor + let alpha = 0.75; + if let Some(ema) = self.bitrate_ema { + let sigma: f64 = (self.target_bitrate_on_delay as f64) - ema; + self.bitrate_ema = Some(ema + (alpha * sigma)); + self.bitrate_emvar = + (1. - alpha) * (self.bitrate_emvar + alpha * sigma.powi(2)); + } else { + self.bitrate_ema = Some(self.target_bitrate_on_delay as f64); + self.bitrate_emvar = 0.; + } + } + } + } + + let target_bitrate = + i32::min(self.target_bitrate_on_delay, self.target_bitrate_on_loss).clamp( + self.min_bitrate as i32 * n_encoders, + self.max_bitrate as i32 * n_encoders, + ) / n_encoders; + + if target_bitrate != prev_bitrate { + gst::info!( + CAT, + "{:?} {} => {} | on delay {} - on loss {} | min {} - max {}", + control_op, + human_bytes::human_bytes(prev_bitrate), + human_bytes::human_bytes(target_bitrate), + human_bytes::human_bytes(self.target_bitrate_on_delay), + human_bytes::human_bytes(self.target_bitrate_on_loss), + human_bytes::human_bytes(self.min_bitrate), + human_bytes::human_bytes(self.max_bitrate), + ); + } + + let fec_ratio = { + if target_bitrate <= 2000000 || self.max_bitrate <= 2000000 { + 0f64 + } else { + (target_bitrate as f64 - 2000000f64) / (self.max_bitrate as f64 - 2000000f64) + } + }; + + let fec_percentage = (fec_ratio * 50f64) as u32; + + for encoder in encoders.iter_mut() { + encoder.set_bitrate(element, target_bitrate); + encoder + .transceiver + .set_property("fec-percentage", fec_percentage); + } + } +} diff --git a/net/webrtc/plugins/src/webrtcsink/imp.rs b/net/webrtc/plugins/src/webrtcsink/imp.rs new file mode 100644 index 00000000..07f773a6 --- /dev/null +++ b/net/webrtc/plugins/src/webrtcsink/imp.rs @@ -0,0 +1,2852 @@ +use anyhow::Context; +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst_rtp::prelude::*; +use gst_utils::StreamProducer; +use gst_video::subclass::prelude::*; +use gst_webrtc::WebRTCDataChannel; + +use async_std::task; +use futures::prelude::*; + +use anyhow::{anyhow, Error}; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::ops::Mul; +use std::sync::Mutex; + +use super::homegrown_cc::CongestionController; +use super::{WebRTCSinkCongestionControl, WebRTCSinkError, WebRTCSinkMitigationMode}; +use crate::signaller::Signaller; +use std::collections::BTreeMap; + +static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { + gst::DebugCategory::new( + "webrtcsink", + gst::DebugColorFlags::empty(), + Some("WebRTC sink"), + ) +}); + +const CUDA_MEMORY_FEATURE: &str = "memory:CUDAMemory"; +const GL_MEMORY_FEATURE: &str = "memory:GLMemory"; +const NVMM_MEMORY_FEATURE: &str = "memory:NVMM"; + +const RTP_TWCC_URI: &str = + "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"; + +const DEFAULT_STUN_SERVER: Option<&str> = Some("stun://stun.l.google.com:19302"); +const DEFAULT_MIN_BITRATE: u32 = 1000; + +/* I have found higher values to cause packet loss *somewhere* in + * my local network, possibly related to chrome's pretty low UDP + * buffer sizes */ +const DEFAULT_MAX_BITRATE: u32 = 8192000; +const DEFAULT_CONGESTION_CONTROL: WebRTCSinkCongestionControl = + WebRTCSinkCongestionControl::GoogleCongestionControl; +const DEFAULT_DO_FEC: bool = true; +const DEFAULT_DO_RETRANSMISSION: bool = true; +const DEFAULT_ENABLE_DATA_CHANNEL_NAVIGATION: bool = false; +const DEFAULT_START_BITRATE: u32 = 2048000; +/* Start adding some FEC when the bitrate > 2Mbps as we found experimentally + * that it is not worth it below that threshold */ +const DO_FEC_THRESHOLD: u32 = 2000000; + +#[derive(Debug, Clone, Copy)] +struct CCInfo { + heuristic: WebRTCSinkCongestionControl, + min_bitrate: u32, + max_bitrate: u32, + start_bitrate: u32, +} + +/// User configuration +struct Settings { + video_caps: gst::Caps, + audio_caps: gst::Caps, + turn_server: Option<String>, + stun_server: Option<String>, + cc_info: CCInfo, + do_fec: bool, + do_retransmission: bool, + enable_data_channel_navigation: bool, + meta: Option<gst::Structure>, +} + +/// Represents a codec we can offer +#[derive(Debug, Clone)] +struct Codec { + encoder: gst::ElementFactory, + payloader: gst::ElementFactory, + caps: gst::Caps, + payload: i32, +} + +impl Codec { + fn is_video(&self) -> bool { + self.encoder + .has_type(gst::ElementFactoryType::VIDEO_ENCODER) + } +} + +/// Wrapper around our sink pads +#[derive(Debug, Clone)] +struct InputStream { + sink_pad: gst::GhostPad, + producer: Option<StreamProducer>, + /// The (fixed) caps coming in + in_caps: Option<gst::Caps>, + /// The caps we will offer, as a set of fixed structures + out_caps: Option<gst::Caps>, + /// Pace input data + clocksync: Option<gst::Element>, +} + +/// Wrapper around webrtcbin pads +#[derive(Clone)] +struct WebRTCPad { + pad: gst::Pad, + /// The (fixed) caps of the corresponding input stream + in_caps: gst::Caps, + /// The m= line index in the SDP + media_idx: u32, + ssrc: u32, + /// The name of the corresponding InputStream's sink_pad + stream_name: String, + /// The payload selected in the answer, None at first + payload: Option<i32>, +} + +/// Wrapper around GStreamer encoder element, keeps track of factory +/// name in order to provide a unified set / get bitrate API, also +/// tracks a raw capsfilter used to resize / decimate the input video +/// stream according to the bitrate, thresholds hardcoded for now +pub struct VideoEncoder { + factory_name: String, + codec_name: String, + element: gst::Element, + filter: gst::Element, + halved_framerate: gst::Fraction, + video_info: gst_video::VideoInfo, + session_id: String, + mitigation_mode: WebRTCSinkMitigationMode, + pub transceiver: gst_webrtc::WebRTCRTPTransceiver, +} + +struct Session { + id: String, + + pipeline: gst::Pipeline, + webrtcbin: gst::Element, + rtprtxsend: Option<gst::Element>, + webrtc_pads: HashMap<u32, WebRTCPad>, + peer_id: String, + encoders: Vec<VideoEncoder>, + + // Our Homegrown controller (if cc_info.heuristic == Homegrown) + congestion_controller: Option<CongestionController>, + // Our BandwidthEstimator (if cc_info.heuristic == GoogleCongestionControl) + rtpgccbwe: Option<gst::Element>, + + sdp: Option<gst_sdp::SDPMessage>, + stats: gst::Structure, + + cc_info: CCInfo, + + links: HashMap<u32, gst_utils::ConsumptionLink>, + stats_sigid: Option<glib::SignalHandlerId>, +} + +#[derive(PartialEq)] +enum SignallerState { + Started, + Stopped, +} + +#[derive(Debug, serde::Deserialize)] +struct NavigationEvent { + mid: Option<String>, + #[serde(flatten)] + event: gst_video::NavigationEvent, +} + +/* Our internal state */ +struct State { + signaller: Box<dyn super::SignallableObject>, + signaller_state: SignallerState, + sessions: HashMap<String, Session>, + codecs: BTreeMap<i32, Codec>, + /// Used to abort codec discovery + codecs_abort_handle: Option<futures::future::AbortHandle>, + /// Used to wait for the discovery task to fully stop + codecs_done_receiver: Option<futures::channel::oneshot::Receiver<()>>, + /// Used to determine whether we can start the signaller when going to Playing, + /// or whether we should wait + codec_discovery_done: bool, + audio_serial: u32, + video_serial: u32, + streams: HashMap<String, InputStream>, + navigation_handler: Option<NavigationEventHandler>, + mids: HashMap<String, String>, +} + +fn create_navigation_event(sink: &super::WebRTCSink, msg: &str) { + let event: Result<NavigationEvent, _> = serde_json::from_str(msg); + + if let Ok(event) = event { + gst::log!(CAT, obj: sink, "Processing navigation event: {:?}", event); + + if let Some(mid) = event.mid { + let this = WebRTCSink::from_instance(sink); + + let state = this.state.lock().unwrap(); + if let Some(stream_name) = state.mids.get(&mid) { + if let Some(stream) = state.streams.get(stream_name) { + let event = gst::event::Navigation::new(event.event.structure()); + + if !stream.sink_pad.push_event(event.clone()) { + gst::info!(CAT, "Could not send event: {:?}", event); + } + } + } + } else { + let this = WebRTCSink::from_instance(sink); + + let state = this.state.lock().unwrap(); + let event = gst::event::Navigation::new(event.event.structure()); + state.streams.iter().for_each(|(_, stream)| { + if stream.sink_pad.name().starts_with("video_") { + gst::log!(CAT, "Navigating to: {:?}", event); + if !stream.sink_pad.push_event(event.clone()) { + gst::info!(CAT, "Could not send event: {:?}", event); + } + } + }); + } + } else { + gst::error!(CAT, "Invalid navigation event: {:?}", msg); + } +} + +/// Wrapper around `gst::ElementFactory::make` with a better error +/// message +pub fn make_element(element: &str, name: Option<&str>) -> Result<gst::Element, Error> { + gst::ElementFactory::make(element, name) + .with_context(|| format!("Failed to make element {}", element)) +} + +/// Simple utility for tearing down a pipeline cleanly +struct PipelineWrapper(gst::Pipeline); + +// Structure to generate GstNavigation event from a WebRTCDataChannel +// This is simply used to hold references to the inner items. +#[derive(Debug)] +struct NavigationEventHandler((glib::SignalHandlerId, WebRTCDataChannel)); + +/// Our instance structure +#[derive(Default)] +pub struct WebRTCSink { + state: Mutex<State>, + settings: Mutex<Settings>, +} + +impl Default for Settings { + fn default() -> Self { + Self { + video_caps: ["video/x-vp8", "video/x-h264", "video/x-vp9", "video/x-h265"] + .iter() + .map(|s| gst::Structure::new_empty(s)) + .collect::<gst::Caps>(), + audio_caps: ["audio/x-opus"] + .iter() + .map(|s| gst::Structure::new_empty(s)) + .collect::<gst::Caps>(), + stun_server: DEFAULT_STUN_SERVER.map(String::from), + turn_server: None, + cc_info: CCInfo { + heuristic: WebRTCSinkCongestionControl::GoogleCongestionControl, + min_bitrate: DEFAULT_MIN_BITRATE, + max_bitrate: DEFAULT_MAX_BITRATE as u32, + start_bitrate: DEFAULT_START_BITRATE, + }, + do_fec: DEFAULT_DO_FEC, + do_retransmission: DEFAULT_DO_RETRANSMISSION, + enable_data_channel_navigation: DEFAULT_ENABLE_DATA_CHANNEL_NAVIGATION, + meta: None, + } + } +} + +impl Default for State { + fn default() -> Self { + let signaller = Signaller::default(); + + Self { + signaller: Box::new(signaller), + signaller_state: SignallerState::Stopped, + sessions: HashMap::new(), + codecs: BTreeMap::new(), + codecs_abort_handle: None, + codecs_done_receiver: None, + codec_discovery_done: false, + audio_serial: 0, + video_serial: 0, + streams: HashMap::new(), + navigation_handler: None, + mids: HashMap::new(), + } + } +} + +fn make_converter_for_video_caps(caps: &gst::Caps) -> Result<gst::Element, Error> { + assert!(caps.is_fixed()); + + let video_info = gst_video::VideoInfo::from_caps(&caps)?; + + let ret = gst::Bin::new(None); + + let (head, mut tail) = { + if let Some(feature) = caps.features(0) { + if feature.contains(CUDA_MEMORY_FEATURE) { + let cudaupload = make_element("cudaupload", None)?; + let cudaconvert = make_element("cudaconvert", None)?; + let cudascale = make_element("cudascale", None)?; + + ret.add_many(&[&cudaupload, &cudaconvert, &cudascale])?; + gst::Element::link_many(&[&cudaupload, &cudaconvert, &cudascale])?; + + (cudaupload, cudascale) + } else if feature.contains(GL_MEMORY_FEATURE) { + let glupload = make_element("glupload", None)?; + let glconvert = make_element("glcolorconvert", None)?; + let glscale = make_element("glcolorscale", None)?; + + ret.add_many(&[&glupload, &glconvert, &glscale])?; + gst::Element::link_many(&[&glupload, &glconvert, &glscale])?; + + (glupload, glscale) + } else if feature.contains(NVMM_MEMORY_FEATURE) { + let queue = make_element("queue", None)?; + let nvconvert = make_element("nvvideoconvert", None)?; + nvconvert.set_property("compute-hw", 0); + nvconvert.set_property("nvbuf-memory-type", 0); + + ret.add_many(&[&queue, &nvconvert])?; + gst::Element::link_many(&[&queue, &nvconvert])?; + + (queue, nvconvert) + } else { + let convert = make_element("videoconvert", None)?; + let scale = make_element("videoscale", None)?; + + ret.add_many(&[&convert, &scale])?; + gst::Element::link_many(&[&convert, &scale])?; + + (convert, scale) + } + } else { + let convert = make_element("videoconvert", None)?; + let scale = make_element("videoscale", None)?; + + ret.add_many(&[&convert, &scale])?; + gst::Element::link_many(&[&convert, &scale])?; + + (convert, scale) + } + }; + + ret.add_pad( + &gst::GhostPad::with_target(Some("sink"), &head.static_pad("sink").unwrap()).unwrap(), + ) + .unwrap(); + + if video_info.fps().numer() != 0 { + let vrate = make_element("videorate", None)?; + vrate.set_property("drop-only", true); + vrate.set_property("skip-to-first", true); + + ret.add(&vrate)?; + tail.link(&vrate)?; + tail = vrate; + } + + ret.add_pad( + &gst::GhostPad::with_target(Some("src"), &tail.static_pad("src").unwrap()).unwrap(), + ) + .unwrap(); + + Ok(ret.upcast()) +} + +/// Default configuration for known encoders, can be disabled +/// by returning True from an encoder-setup handler. +fn configure_encoder(enc: &gst::Element, start_bitrate: u32) { + if let Some(factory) = enc.factory() { + match factory.name().as_str() { + "vp8enc" | "vp9enc" => { + enc.set_property("deadline", 1i64); + enc.set_property("target-bitrate", start_bitrate as i32); + enc.set_property("cpu-used", -16i32); + enc.set_property("keyframe-max-dist", 2000i32); + enc.set_property_from_str("keyframe-mode", "disabled"); + enc.set_property_from_str("end-usage", "cbr"); + enc.set_property("buffer-initial-size", 100i32); + enc.set_property("buffer-optimal-size", 120i32); + enc.set_property("buffer-size", 150i32); + enc.set_property("max-intra-bitrate", 250i32); + enc.set_property_from_str("error-resilient", "default"); + enc.set_property("lag-in-frames", 0i32); + } + "x264enc" => { + enc.set_property("bitrate", start_bitrate / 1000); + enc.set_property_from_str("tune", "zerolatency"); + enc.set_property_from_str("speed-preset", "ultrafast"); + enc.set_property("threads", 12u32); + enc.set_property("key-int-max", 2560u32); + enc.set_property("b-adapt", false); + enc.set_property("vbv-buf-capacity", 120u32); + } + "nvh264enc" => { + enc.set_property("bitrate", start_bitrate / 1000); + enc.set_property("gop-size", 2560i32); + enc.set_property_from_str("rc-mode", "cbr-ld-hq"); + enc.set_property("zerolatency", true); + } + "vaapih264enc" | "vaapivp8enc" => { + enc.set_property("bitrate", start_bitrate / 1000); + enc.set_property("keyframe-period", 2560u32); + enc.set_property_from_str("rate-control", "cbr"); + } + "nvv4l2h264enc" => { + enc.set_property("bitrate", start_bitrate); + enc.set_property_from_str("preset-level", "UltraFastPreset"); + enc.set_property("maxperf-enable", true); + enc.set_property("insert-vui", true); + enc.set_property("idrinterval", 256u32); + enc.set_property("insert-sps-pps", true); + enc.set_property("insert-aud", true); + enc.set_property_from_str("control-rate", "constant_bitrate"); + } + "nvv4l2vp8enc" => { + enc.set_property("bitrate", start_bitrate); + enc.set_property_from_str("preset-level", "UltraFastPreset"); + enc.set_property("maxperf-enable", true); + enc.set_property("idrinterval", 256u32); + enc.set_property_from_str("control-rate", "constant_bitrate"); + } + _ => (), + } + } +} + +/// Bit of an awkward function, but the goal here is to keep +/// most of the encoding code for consumers in line with +/// the codec discovery code, and this gets the job done. +fn setup_encoding( + pipeline: &gst::Pipeline, + src: &gst::Element, + input_caps: &gst::Caps, + codec: &Codec, + ssrc: Option<u32>, + twcc: bool, +) -> Result<(gst::Element, gst::Element, gst::Element), Error> { + let conv = match codec.is_video() { + true => make_converter_for_video_caps(input_caps)?.upcast(), + false => gst::parse_bin_from_description("audioresample ! audioconvert", true)?.upcast(), + }; + + let conv_filter = make_element("capsfilter", None)?; + + let enc = codec + .encoder + .create(None) + .with_context(|| format!("Creating encoder {}", codec.encoder.name()))?; + let pay = codec + .payloader + .create(None) + .with_context(|| format!("Creating payloader {}", codec.payloader.name()))?; + let parse_filter = make_element("capsfilter", None)?; + + pay.set_property("mtu", 1200 as u32); + pay.set_property("pt", codec.payload as u32); + + if let Some(ssrc) = ssrc { + pay.set_property("ssrc", ssrc); + } + + pipeline + .add_many(&[&conv, &conv_filter, &enc, &parse_filter, &pay]) + .unwrap(); + gst::Element::link_many(&[src, &conv, &conv_filter, &enc]) + .with_context(|| "Linking encoding elements")?; + + let codec_name = codec.caps.structure(0).unwrap().name(); + + if let Some(parser) = if codec_name == "video/x-h264" { + Some(make_element("h264parse", None)?) + } else if codec_name == "video/x-h265" { + Some(make_element("h265parse", None)?) + } else { + None + } { + pipeline.add(&parser).unwrap(); + gst::Element::link_many(&[&enc, &parser, &parse_filter]) + .with_context(|| "Linking encoding elements")?; + } else { + gst::Element::link_many(&[&enc, &parse_filter]) + .with_context(|| "Linking encoding elements")?; + } + + let conv_caps = if codec.is_video() { + let mut structure_builder = gst::Structure::builder("video/x-raw") + .field("pixel-aspect-ratio", gst::Fraction::new(1, 1)); + + if codec.encoder.name() == "nvh264enc" { + // Quirk: nvh264enc can perform conversion from RGB formats, but + // doesn't advertise / negotiate colorimetry correctly, leading + // to incorrect color display in Chrome (but interestingly not in + // Firefox). In any case, restrict to exclude RGB formats altogether, + // and let videoconvert do the conversion properly if needed. + structure_builder = + structure_builder.field("format", &gst::List::new(&[&"NV12", &"YV12", &"I420"])); + } + + gst::Caps::builder_full_with_any_features() + .structure(structure_builder.build()) + .build() + } else { + gst::Caps::builder("audio/x-raw").build() + }; + + match codec.encoder.name().as_str() { + "vp8enc" | "vp9enc" => { + pay.set_property_from_str("picture-id-mode", "15-bit"); + } + _ => (), + } + + /* We only enforce TWCC in the offer caps, once a remote description + * has been set it will get automatically negotiated. This is necessary + * because the implementor in Firefox had apparently not understood the + * concept of *transport-wide* congestion control, and firefox doesn't + * provide feedback for audio packets. + */ + if twcc { + let twcc_extension = gst_rtp::RTPHeaderExtension::create_from_uri(RTP_TWCC_URI).unwrap(); + twcc_extension.set_id(1); + pay.emit_by_name::<()>("add-extension", &[&twcc_extension]); + } + + conv_filter.set_property("caps", conv_caps); + + let parse_caps = if codec_name == "video/x-h264" { + gst::Caps::builder(codec_name) + .field("stream-format", "avc") + .field("profile", "constrained-baseline") + .build() + } else if codec_name == "video/x-h265" { + gst::Caps::builder(codec_name) + .field("stream-format", "hvc1") + .build() + } else { + gst::Caps::new_any() + }; + + parse_filter.set_property("caps", parse_caps); + + gst::Element::link_many(&[&parse_filter, &pay]).with_context(|| "Linking encoding elements")?; + + Ok((enc, conv_filter, pay)) +} + +impl VideoEncoder { + fn new( + element: gst::Element, + filter: gst::Element, + video_info: gst_video::VideoInfo, + peer_id: &str, + codec_name: &str, + transceiver: gst_webrtc::WebRTCRTPTransceiver, + ) -> Self { + let halved_framerate = video_info.fps().mul(gst::Fraction::new(1, 2)); + + Self { + factory_name: element.factory().unwrap().name().into(), + codec_name: codec_name.to_string(), + element, + filter, + halved_framerate, + video_info, + session_id: peer_id.to_string(), + mitigation_mode: WebRTCSinkMitigationMode::NONE, + transceiver, + } + } + + pub fn bitrate(&self) -> i32 { + match self.factory_name.as_str() { + "vp8enc" | "vp9enc" => self.element.property::<i32>("target-bitrate"), + "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" => { + (self.element.property::<u32>("bitrate") * 1000) as i32 + } + "nvv4l2h264enc" | "nvv4l2vp8enc" => (self.element.property::<u32>("bitrate")) as i32, + factory => unimplemented!("Factory {} is currently not supported", factory), + } + } + + pub fn scale_height_round_2(&self, height: i32) -> i32 { + let ratio = gst_video::calculate_display_ratio( + self.video_info.width(), + self.video_info.height(), + self.video_info.par(), + gst::Fraction::new(1, 1), + ) + .unwrap(); + + let width = height.mul_div_ceil(ratio.numer(), ratio.denom()).unwrap(); + + (width + 1) & !1 + } + + pub fn set_bitrate(&mut self, element: &super::WebRTCSink, bitrate: i32) { + match self.factory_name.as_str() { + "vp8enc" | "vp9enc" => self.element.set_property("target-bitrate", bitrate), + "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" => self + .element + .set_property("bitrate", (bitrate / 1000) as u32), + "nvv4l2h264enc" | "nvv4l2vp8enc" => { + self.element.set_property("bitrate", bitrate as u32) + } + factory => unimplemented!("Factory {} is currently not supported", factory), + } + + let current_caps = self.filter.property::<gst::Caps>("caps"); + let mut s = current_caps.structure(0).unwrap().to_owned(); + + // Hardcoded thresholds, may be tuned further in the future, and + // adapted according to the codec in use + if bitrate < 500000 { + let height = 360i32.min(self.video_info.height() as i32); + let width = self.scale_height_round_2(height); + + s.set("height", height); + s.set("width", width); + + if self.halved_framerate.numer() != 0 { + s.set("framerate", self.halved_framerate); + } + + self.mitigation_mode = + WebRTCSinkMitigationMode::DOWNSAMPLED | WebRTCSinkMitigationMode::DOWNSCALED; + } else if bitrate < 1000000 { + let height = 360i32.min(self.video_info.height() as i32); + let width = self.scale_height_round_2(height); + + s.set("height", height); + s.set("width", width); + s.remove_field("framerate"); + + self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED; + } else if bitrate < 2000000 { + let height = 720i32.min(self.video_info.height() as i32); + let width = self.scale_height_round_2(height); + + s.set("height", height); + s.set("width", width); + s.remove_field("framerate"); + + self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED; + } else { + s.remove_field("height"); + s.remove_field("width"); + s.remove_field("framerate"); + + self.mitigation_mode = WebRTCSinkMitigationMode::NONE; + } + + let caps = gst::Caps::builder_full_with_any_features() + .structure(s) + .build(); + + if !caps.is_strictly_equal(¤t_caps) { + gst::log!( + CAT, + obj: element, + "session {}: setting bitrate {} and caps {} on encoder {:?}", + self.session_id, + bitrate, + caps, + self.element + ); + + self.filter.set_property("caps", caps); + } + } + + fn gather_stats(&self) -> gst::Structure { + gst::Structure::builder("application/x-webrtcsink-video-encoder-stats") + .field("bitrate", self.bitrate()) + .field("mitigation-mode", self.mitigation_mode) + .field("codec-name", self.codec_name.as_str()) + .field( + "fec-percentage", + self.transceiver.property::<u32>("fec-percentage"), + ) + .build() + } +} + +impl State { + fn finalize_session( + &mut self, + element: &super::WebRTCSink, + session: &mut Session, + signal: bool, + ) { + gst::info!(CAT, "Ending session {}", session.id); + session.pipeline.debug_to_dot_file_with_ts( + gst::DebugGraphDetails::all(), + format!("removing-peer-{}-", session.peer_id,), + ); + + for ssrc in session.webrtc_pads.keys() { + session.links.remove(ssrc); + } + + session.pipeline.call_async(|pipeline| { + let _ = pipeline.set_state(gst::State::Null); + }); + + if signal { + self.signaller.session_ended(element, &session.peer_id); + } + } + + fn end_session( + &mut self, + element: &super::WebRTCSink, + session_id: &str, + signal: bool, + ) -> Option<Session> { + if let Some(mut session) = self.sessions.remove(session_id) { + self.finalize_session(element, &mut session, signal); + Some(session) + } else { + None + } + } + + fn maybe_start_signaller(&mut self, element: &super::WebRTCSink) { + if self.signaller_state == SignallerState::Stopped + && element.current_state() >= gst::State::Paused + && self.codec_discovery_done + { + if let Err(err) = self.signaller.start(element) { + gst::error!(CAT, obj: element, "error: {}", err); + gst::element_error!( + element, + gst::StreamError::Failed, + ["Failed to start signaller {}", err] + ); + } else { + gst::info!(CAT, "Started signaller"); + self.signaller_state = SignallerState::Started; + } + } + } + + fn maybe_stop_signaller(&mut self, element: &super::WebRTCSink) { + if self.signaller_state == SignallerState::Started { + self.signaller.stop(element); + self.signaller_state = SignallerState::Stopped; + gst::info!(CAT, "Stopped signaller"); + } + } +} + +impl Session { + fn new( + id: String, + pipeline: gst::Pipeline, + webrtcbin: gst::Element, + peer_id: String, + congestion_controller: Option<CongestionController>, + rtpgccbwe: Option<gst::Element>, + cc_info: CCInfo, + ) -> Self { + Self { + id, + pipeline, + webrtcbin, + peer_id, + cc_info, + rtprtxsend: None, + congestion_controller, + rtpgccbwe, + stats: gst::Structure::new_empty("application/x-webrtc-stats"), + sdp: None, + webrtc_pads: HashMap::new(), + encoders: Vec::new(), + links: HashMap::new(), + stats_sigid: None, + } + } + + fn gather_stats(&self) -> gst::Structure { + let mut ret = self.stats.to_owned(); + + let encoder_stats: Vec<_> = self + .encoders + .iter() + .map(VideoEncoder::gather_stats) + .map(|s| s.to_send_value()) + .collect(); + + let our_stats = gst::Structure::builder("application/x-webrtcsink-consumer-stats") + .field("video-encoders", gst::Array::from(encoder_stats)) + .build(); + + ret.set("consumer-stats", our_stats); + + ret + } + + fn generate_ssrc(&self) -> u32 { + loop { + let ret = fastrand::u32(..); + + if !self.webrtc_pads.contains_key(&ret) { + return ret; + } + } + } + + /// Request a sink pad on our webrtcbin, and set its transceiver's codec_preferences + fn request_webrtcbin_pad( + &mut self, + element: &super::WebRTCSink, + settings: &Settings, + stream: &InputStream, + ) { + let ssrc = self.generate_ssrc(); + let media_idx = self.webrtc_pads.len() as i32; + + let mut payloader_caps = stream.out_caps.as_ref().unwrap().to_owned(); + + { + let payloader_caps_mut = payloader_caps.make_mut(); + payloader_caps_mut.set_simple(&[("ssrc", &ssrc)]); + } + + gst::info!( + CAT, + obj: element, + "Requesting WebRTC pad for consumer {} with caps {}", + self.peer_id, + payloader_caps + ); + + let pad = self + .webrtcbin + .request_pad_simple(&format!("sink_{}", media_idx)) + .unwrap(); + + let transceiver = pad.property::<gst_webrtc::WebRTCRTPTransceiver>("transceiver"); + + transceiver.set_property( + "direction", + gst_webrtc::WebRTCRTPTransceiverDirection::Sendonly, + ); + + transceiver.set_property("codec-preferences", &payloader_caps); + + if stream.sink_pad.name().starts_with("video_") { + if settings.do_fec { + transceiver.set_property("fec-type", gst_webrtc::WebRTCFECType::UlpRed); + } + + transceiver.set_property("do-nack", settings.do_retransmission); + } + + self.webrtc_pads.insert( + ssrc, + WebRTCPad { + pad, + in_caps: stream.in_caps.as_ref().unwrap().clone(), + media_idx: media_idx as u32, + ssrc, + stream_name: stream.sink_pad.name().to_string(), + payload: None, + }, + ); + } + + /// Called when we have received an answer, connects an InputStream + /// to a given WebRTCPad + fn connect_input_stream( + &mut self, + element: &super::WebRTCSink, + producer: &StreamProducer, + webrtc_pad: &WebRTCPad, + codecs: &BTreeMap<i32, Codec>, + ) -> Result<(), Error> { + gst::info!( + CAT, + obj: element, + "Connecting input stream {} for consumer {}", + webrtc_pad.stream_name, + self.peer_id + ); + + let payload = webrtc_pad.payload.unwrap(); + + let codec = codecs + .get(&payload) + .ok_or_else(|| anyhow!("No codec for payload {}", payload))?; + + let appsrc = make_element("appsrc", Some(&webrtc_pad.stream_name))?; + self.pipeline.add(&appsrc).unwrap(); + + let pay_filter = make_element("capsfilter", None)?; + self.pipeline.add(&pay_filter).unwrap(); + + let (enc, raw_filter, pay) = setup_encoding( + &self.pipeline, + &appsrc, + &webrtc_pad.in_caps, + codec, + Some(webrtc_pad.ssrc), + false, + )?; + + element.emit_by_name::<bool>( + "encoder-setup", + &[&self.peer_id, &webrtc_pad.stream_name, &enc], + ); + + // At this point, the peer has provided its answer, and we want to + // let the payloader / encoder perform negotiation according to that. + // + // This means we need to unset our codec preferences, as they would now + // conflict with what the peer actually requested (see webrtcbin's + // caps query implementation), and instead install a capsfilter downstream + // of the payloader with caps constructed from the relevant SDP media. + let transceiver = webrtc_pad + .pad + .property::<gst_webrtc::WebRTCRTPTransceiver>("transceiver"); + transceiver.set_property("codec-preferences", None::<gst::Caps>); + + let mut global_caps = gst::Caps::new_simple("application/x-unknown", &[]); + + let sdp = self.sdp.as_ref().unwrap(); + let sdp_media = sdp.media(webrtc_pad.media_idx).unwrap(); + + sdp.attributes_to_caps(global_caps.get_mut().unwrap()) + .unwrap(); + sdp_media + .attributes_to_caps(global_caps.get_mut().unwrap()) + .unwrap(); + + let caps = sdp_media + .caps_from_media(payload) + .unwrap() + .intersect(&global_caps); + let s = caps.structure(0).unwrap(); + let mut filtered_s = gst::Structure::new_empty("application/x-rtp"); + + filtered_s.extend(s.iter().filter_map(|(key, value)| { + if key.starts_with("a-") { + None + } else { + Some((key, value.to_owned())) + } + })); + filtered_s.set("ssrc", webrtc_pad.ssrc); + + let caps = gst::Caps::builder_full().structure(filtered_s).build(); + + pay_filter.set_property("caps", caps); + + if codec.is_video() { + let video_info = gst_video::VideoInfo::from_caps(&webrtc_pad.in_caps)?; + let mut enc = VideoEncoder::new( + enc, + raw_filter, + video_info, + &self.peer_id, + codec.caps.structure(0).unwrap().name(), + transceiver, + ); + + match self.cc_info.heuristic { + WebRTCSinkCongestionControl::Disabled => { + // If congestion control is disabled, we simply use the highest + // known "safe" value for the bitrate. + enc.set_bitrate(element, self.cc_info.max_bitrate as i32); + enc.transceiver.set_property("fec-percentage", 50u32); + } + WebRTCSinkCongestionControl::Homegrown => { + if let Some(congestion_controller) = self.congestion_controller.as_mut() { + congestion_controller.target_bitrate_on_delay += enc.bitrate(); + congestion_controller.target_bitrate_on_loss = + congestion_controller.target_bitrate_on_delay; + enc.transceiver.set_property("fec-percentage", 0u32); + } else { + /* If congestion control is disabled, we simply use the highest + * known "safe" value for the bitrate. */ + enc.set_bitrate(element, self.cc_info.max_bitrate as i32); + enc.transceiver.set_property("fec-percentage", 50u32); + } + } + _ => enc.transceiver.set_property("fec-percentage", 0u32), + } + + self.encoders.push(enc); + + if let Some(ref rtpgccbwe) = self.rtpgccbwe.as_ref() { + let max_bitrate = self.cc_info.max_bitrate * (self.encoders.len() as u32); + rtpgccbwe.set_property("max-bitrate", max_bitrate); + } + } + + let appsrc = appsrc.downcast::<gst_app::AppSrc>().unwrap(); + gst_utils::StreamProducer::configure_consumer(&appsrc); + self.pipeline + .sync_children_states() + .with_context(|| format!("Connecting input stream for {}", self.peer_id))?; + + pay.link(&pay_filter)?; + + let srcpad = pay_filter.static_pad("src").unwrap(); + + srcpad + .link(&webrtc_pad.pad) + .with_context(|| format!("Connecting input stream for {}", self.peer_id))?; + + match producer.add_consumer(&appsrc) { + Ok(link) => { + self.links.insert(webrtc_pad.ssrc, link); + Ok(()) + } + Err(err) => Err(anyhow!("Could not link producer: {:?}", err)), + } + } +} + +impl Drop for PipelineWrapper { + fn drop(&mut self) { + let _ = self.0.set_state(gst::State::Null); + } +} + +impl InputStream { + /// Called when transitioning state up to Paused + fn prepare(&mut self, element: &super::WebRTCSink) -> Result<(), Error> { + let clocksync = make_element("clocksync", None)?; + let appsink = make_element("appsink", None)? + .downcast::<gst_app::AppSink>() + .unwrap(); + + element.add(&clocksync).unwrap(); + element.add(&appsink).unwrap(); + + clocksync + .link(&appsink) + .with_context(|| format!("Linking input stream {}", self.sink_pad.name()))?; + + element + .sync_children_states() + .with_context(|| format!("Linking input stream {}", self.sink_pad.name()))?; + + self.sink_pad + .set_target(Some(&clocksync.static_pad("sink").unwrap())) + .unwrap(); + + let producer = StreamProducer::from(&appsink); + producer.forward(); + + self.producer = Some(producer); + + Ok(()) + } + + /// Called when transitioning state back down to Ready + fn unprepare(&mut self, element: &super::WebRTCSink) { + self.sink_pad.set_target(None::<&gst::Pad>).unwrap(); + + if let Some(clocksync) = self.clocksync.take() { + element.remove(&clocksync).unwrap(); + clocksync.set_state(gst::State::Null).unwrap(); + } + + if let Some(producer) = self.producer.take() { + let appsink = producer.appsink().upcast_ref::<gst::Element>(); + element.remove(appsink).unwrap(); + appsink.set_state(gst::State::Null).unwrap(); + } + } +} + +impl NavigationEventHandler { + pub fn new(element: &super::WebRTCSink, webrtcbin: &gst::Element) -> Self { + gst::info!(CAT, "Creating navigation data channel"); + let channel = webrtcbin.emit_by_name::<WebRTCDataChannel>( + "create-data-channel", + &[ + &"input", + &gst::Structure::new( + "config", + &[("priority", &gst_webrtc::WebRTCPriorityType::High)], + ), + ], + ); + + let weak_element = element.downgrade(); + Self(( + channel.connect("on-message-string", false, move |values| { + if let Some(element) = weak_element.upgrade() { + let _channel = values[0].get::<WebRTCDataChannel>().unwrap(); + let msg = values[1].get::<&str>().unwrap(); + create_navigation_event(&element, msg); + } + + None + }), + channel, + )) + } +} + +impl WebRTCSink { + /// Build an ordered map of Codecs, given user-provided audio / video caps */ + fn lookup_codecs(&self) -> BTreeMap<i32, Codec> { + /* First gather all encoder and payloader factories */ + let encoders = gst::ElementFactory::factories_with_type( + gst::ElementFactoryType::ENCODER, + gst::Rank::Marginal, + ); + + let payloaders = gst::ElementFactory::factories_with_type( + gst::ElementFactoryType::PAYLOADER, + gst::Rank::Marginal, + ); + + /* Now iterate user-provided codec preferences and determine + * whether we can fulfill these preferences */ + let settings = self.settings.lock().unwrap(); + let mut payload = 96..128; + + settings + .video_caps + .iter() + .chain(settings.audio_caps.iter()) + .filter_map(move |s| { + let caps = gst::Caps::builder_full().structure(s.to_owned()).build(); + + Option::zip( + encoders + .iter() + .find(|factory| factory.can_src_any_caps(&caps)), + payloaders + .iter() + .find(|factory| factory.can_sink_any_caps(&caps)), + ) + .and_then(|(encoder, payloader)| { + /* Assign a payload type to the codec */ + if let Some(pt) = payload.next() { + Some(Codec { + encoder: encoder.clone(), + payloader: payloader.clone(), + caps, + payload: pt, + }) + } else { + gst::warning!(CAT, imp: self, + "Too many formats for available payload type range, ignoring {}", + s); + None + } + }) + }) + .map(|codec| (codec.payload, codec)) + .collect() + } + + /// Prepare for accepting consumers, by setting + /// up StreamProducers for each of our sink pads + fn prepare(&self, element: &super::WebRTCSink) -> Result<(), Error> { + gst::debug!(CAT, obj: element, "preparing"); + + self.state + .lock() + .unwrap() + .streams + .iter_mut() + .try_for_each(|(_, stream)| stream.prepare(element))?; + + Ok(()) + } + + /// Unprepare by stopping consumers, then the signaller object. + /// Might abort codec discovery + fn unprepare(&self, element: &super::WebRTCSink) -> Result<(), Error> { + gst::info!(CAT, obj: element, "unpreparing"); + + let mut state = self.state.lock().unwrap(); + + let session_ids: Vec<_> = state.sessions.keys().map(|k| k.to_owned()).collect(); + + for id in session_ids { + state.end_session(element, &id, true); + } + + state + .streams + .iter_mut() + .for_each(|(_, stream)| stream.unprepare(element)); + + if let Some(handle) = state.codecs_abort_handle.take() { + handle.abort(); + } + + if let Some(receiver) = state.codecs_done_receiver.take() { + task::block_on(async { + let _ = receiver.await; + }); + } + + state.maybe_stop_signaller(element); + + state.codec_discovery_done = false; + state.codecs = BTreeMap::new(); + + Ok(()) + } + + /// When using a custom signaller + pub fn set_signaller(&self, signaller: Box<dyn super::SignallableObject>) -> Result<(), Error> { + let mut state = self.state.lock().unwrap(); + + state.signaller = signaller; + + Ok(()) + } + + /// Called by the signaller when it has encountered an error + pub fn handle_signalling_error(&self, element: &super::WebRTCSink, error: anyhow::Error) { + gst::error!(CAT, obj: element, "Signalling error: {:?}", error); + + gst::element_error!( + element, + gst::StreamError::Failed, + ["Signalling error: {:?}", error] + ); + } + + fn on_offer_created( + &self, + element: &super::WebRTCSink, + offer: gst_webrtc::WebRTCSessionDescription, + session_id: &str, + ) { + let mut state = self.state.lock().unwrap(); + + if let Some(session) = state.sessions.get(session_id) { + session + .webrtcbin + .emit_by_name::<()>("set-local-description", &[&offer, &None::<gst::Promise>]); + + if let Err(err) = state.signaller.handle_sdp(element, session_id, &offer) { + gst::warning!( + CAT, + "Failed to handle SDP for session {}: {}", + session_id, + err + ); + + state.end_session(element, session_id, true); + } + } + } + + fn negotiate(&self, element: &super::WebRTCSink, session_id: &str) { + let state = self.state.lock().unwrap(); + + gst::debug!(CAT, obj: element, "Negotiating for session {}", session_id); + + if let Some(session) = state.sessions.get(session_id) { + let element = element.downgrade(); + gst::debug!(CAT, "Creating offer for session {}", session_id); + let session_id = session_id.to_string(); + let promise = gst::Promise::with_change_func(move |reply| { + gst::debug!(CAT, "Created offer for session {}", session_id); + + if let Some(element) = element.upgrade() { + let this = Self::from_instance(&element); + let reply = match reply { + Ok(Some(reply)) => reply, + Ok(None) => { + gst::warning!( + CAT, + obj: &element, + "Promise returned without a reply for {}", + session_id + ); + let _ = this.remove_session(&element, &session_id, true); + return; + } + Err(err) => { + gst::warning!( + CAT, + obj: &element, + "Promise returned with an error for {}: {:?}", + session_id, + err + ); + let _ = this.remove_session(&element, &session_id, true); + return; + } + }; + + if let Ok(offer) = reply + .value("offer") + .map(|offer| offer.get::<gst_webrtc::WebRTCSessionDescription>().unwrap()) + { + this.on_offer_created(&element, offer, &session_id); + } else { + gst::warning!( + CAT, + "Reply without an offer for session {}: {:?}", + session_id, + reply + ); + let _ = this.remove_session(&element, &session_id, true); + } + } + }); + + session + .webrtcbin + .emit_by_name::<()>("create-offer", &[&None::<gst::Structure>, &promise]); + } else { + gst::debug!( + CAT, + obj: element, + "consumer for session {} no longer exists (sessions: {:?}", + session_id, + state.sessions.keys().map(|id| id) + ); + } + } + + fn on_ice_candidate( + &self, + element: &super::WebRTCSink, + session_id: String, + sdp_m_line_index: u32, + candidate: String, + ) { + let mut state = self.state.lock().unwrap(); + if let Err(err) = state.signaller.handle_ice( + element, + &session_id, + &candidate, + Some(sdp_m_line_index), + None, + ) { + gst::warning!( + CAT, + "Failed to handle ICE in session {}: {}", + session_id, + err + ); + + state.end_session(element, &session_id, true); + } + } + + /// Called by the signaller to add a new session + pub fn start_session( + &self, + element: &super::WebRTCSink, + session_id: &str, + peer_id: &str, + ) -> Result<(), WebRTCSinkError> { + let settings = self.settings.lock().unwrap(); + let mut state = self.state.lock().unwrap(); + let peer_id = peer_id.to_string(); + let session_id = session_id.to_string(); + + if state.sessions.contains_key(&session_id) { + return Err(WebRTCSinkError::DuplicateSessionId(session_id)); + } + + gst::info!( + CAT, + obj: element, + "Adding session: {} for peer: {}", + peer_id, + session_id + ); + + let pipeline = gst::Pipeline::new(Some(&format!("session-pipeline-{}", session_id))); + + let webrtcbin = make_element("webrtcbin", Some(&format!("webrtcbin-{}", session_id))) + .map_err(|err| WebRTCSinkError::SessionPipelineError { + session_id: session_id.clone(), + peer_id: peer_id.clone(), + details: err.to_string(), + })?; + + webrtcbin.set_property_from_str("bundle-policy", "max-bundle"); + + if let Some(stun_server) = settings.stun_server.as_ref() { + webrtcbin.set_property("stun-server", stun_server); + } + + if let Some(turn_server) = settings.turn_server.as_ref() { + webrtcbin.set_property("turn-server", turn_server); + } + + let rtpgccbwe = match settings.cc_info.heuristic { + WebRTCSinkCongestionControl::GoogleCongestionControl => { + let rtpgccbwe = match gst::ElementFactory::make("rtpgccbwe", None) { + Err(err) => { + glib::g_warning!( + "webrtcsink", + "The `rtpgccbwe` element is not available \ + not doing any congestion control: {err:?}" + ); + None + } + Ok(cc) => { + webrtcbin.connect_closure( + "request-aux-sender", + false, + glib::closure!(@watch element, @strong session_id, @weak-allow-none cc + => move |_webrtcbin: gst::Element, _transport: gst::Object| { + + let cc = cc.unwrap(); + let settings = element.imp().settings.lock().unwrap(); + + // TODO: Bind properties with @element's + cc.set_properties(&[ + ("min-bitrate", &settings.cc_info.min_bitrate), + ("estimated-bitrate", &settings.cc_info.start_bitrate), + ("max-bitrate", &settings.cc_info.max_bitrate), + ]); + + cc.connect_notify(Some("estimated-bitrate"), + glib::clone!(@weak element, @strong session_id + => move |bwe, pspec| { + element.imp().set_bitrate(&element, &session_id, + bwe.property::<u32>(pspec.name())); + } + )); + + Some(cc) + }), + ); + + Some(cc) + } + }; + + webrtcbin.connect_closure( + "deep-element-added", + false, + glib::closure!(@watch element, @strong session_id + => move |_webrtcbin: gst::Element, _bin: gst::Bin, e: gst::Element| { + + if e.factory().map_or(false, |f| f.name() == "rtprtxsend") { + if e.has_property("stuffing-kbps", Some(i32::static_type())) { + element.imp().set_rtptrxsend(&element, &session_id, e); + } else { + gst::warning!(CAT, "rtprtxsend doesn't have a `stuffing-kbps` \ + property, stuffing disabled"); + } + } + }), + ); + + rtpgccbwe + } + _ => None, + }; + + pipeline.add(&webrtcbin).unwrap(); + + let element_clone = element.downgrade(); + let session_id_clone = session_id.clone(); + webrtcbin.connect("on-ice-candidate", false, move |values| { + if let Some(element) = element_clone.upgrade() { + let this = Self::from_instance(&element); + let sdp_m_line_index = values[1].get::<u32>().expect("Invalid argument"); + let candidate = values[2].get::<String>().expect("Invalid argument"); + this.on_ice_candidate( + &element, + session_id_clone.to_string(), + sdp_m_line_index, + candidate, + ); + } + None + }); + + let element_clone = element.downgrade(); + let peer_id_clone = peer_id.clone(); + let session_id_clone = session_id.clone(); + webrtcbin.connect_notify(Some("connection-state"), move |webrtcbin, _pspec| { + if let Some(element) = element_clone.upgrade() { + let state = + webrtcbin.property::<gst_webrtc::WebRTCPeerConnectionState>("connection-state"); + + match state { + gst_webrtc::WebRTCPeerConnectionState::Failed => { + let this = Self::from_instance(&element); + gst::warning!( + CAT, + obj: &element, + "Connection state for in session {} (peer {}) failed", + session_id_clone, + peer_id_clone + ); + let _ = this.remove_session(&element, &session_id_clone, true); + } + _ => { + gst::log!( + CAT, + obj: &element, + "Connection state in session {} (peer {}) changed: {:?}", + session_id_clone, + peer_id_clone, + state + ); + } + } + } + }); + + let element_clone = element.downgrade(); + let peer_id_clone = peer_id.clone(); + let session_id_clone = session_id.clone(); + webrtcbin.connect_notify(Some("ice-connection-state"), move |webrtcbin, _pspec| { + if let Some(element) = element_clone.upgrade() { + let state = webrtcbin + .property::<gst_webrtc::WebRTCICEConnectionState>("ice-connection-state"); + let this = Self::from_instance(&element); + + match state { + gst_webrtc::WebRTCICEConnectionState::Failed => { + gst::warning!( + CAT, + obj: &element, + "Ice connection state in session {} (peer {}) failed", + session_id_clone, + peer_id_clone, + ); + let _ = this.remove_session(&element, &session_id_clone, true); + } + _ => { + gst::log!( + CAT, + obj: &element, + "Ice connection state in session {} (peer {}) changed: {:?}", + session_id_clone, + peer_id_clone, + state + ); + } + } + + if state == gst_webrtc::WebRTCICEConnectionState::Completed { + let state = this.state.lock().unwrap(); + + if let Some(session) = state.sessions.get(&session_id_clone) { + for webrtc_pad in session.webrtc_pads.values() { + if let Some(srcpad) = webrtc_pad.pad.peer() { + srcpad.send_event( + gst_video::UpstreamForceKeyUnitEvent::builder() + .all_headers(true) + .build(), + ); + } + } + } + } + } + }); + + let element_clone = element.downgrade(); + let peer_id_clone = peer_id.clone(); + let session_id_clone = session_id.clone(); + webrtcbin.connect_notify(Some("ice-gathering-state"), move |webrtcbin, _pspec| { + let state = + webrtcbin.property::<gst_webrtc::WebRTCICEGatheringState>("ice-gathering-state"); + + if let Some(element) = element_clone.upgrade() { + gst::log!( + CAT, + obj: &element, + "Ice gathering state in session {} (peer {}) changed: {:?}", + session_id_clone, + peer_id_clone, + state + ); + } + }); + + let mut session = Session::new( + session_id.clone(), + pipeline.clone(), + webrtcbin.clone(), + peer_id.clone(), + match settings.cc_info.heuristic { + WebRTCSinkCongestionControl::Homegrown => Some(CongestionController::new( + &peer_id, + settings.cc_info.min_bitrate, + settings.cc_info.max_bitrate, + )), + _ => None, + }, + rtpgccbwe, + settings.cc_info, + ); + + let rtpbin = webrtcbin + .dynamic_cast_ref::<gst::ChildProxy>() + .unwrap() + .child_by_name("rtpbin") + .unwrap(); + + if session.congestion_controller.is_some() { + let session_id_str = session_id.to_string(); + if session.stats_sigid.is_none() { + session.stats_sigid = Some(rtpbin.connect_closure("on-new-ssrc", true, + glib::closure!(@weak-allow-none element, @weak-allow-none webrtcbin + => move |rtpbin: gst::Object, session_id: u32, _src: u32| { + let rtp_session = rtpbin.emit_by_name::<gst::Element>("get-session", &[&session_id]); + + let element = element.expect("on-new-ssrc emited when webrtcsink has been disposed?"); + let webrtcbin = webrtcbin.unwrap(); + let mut state = element.imp().state.lock().unwrap(); + if let Some(mut session) = state.sessions.get_mut(&session_id_str) { + + session.stats_sigid = Some(rtp_session.connect_notify(Some("twcc-stats"), + glib::clone!(@strong session_id_str, @weak webrtcbin, @weak element => @default-panic, move |sess, pspec| { + // Run the Loss-based control algorithm on new peer TWCC feedbacks + element.imp().process_loss_stats(&element, &session_id_str, &sess.property::<gst::Structure>(pspec.name())); + }) + )); + } + }) + )); + } + } + + state + .streams + .iter() + .for_each(|(_, stream)| session.request_webrtcbin_pad(element, &settings, stream)); + + let clock = element.clock(); + + pipeline.use_clock(clock.as_ref()); + pipeline.set_start_time(gst::ClockTime::NONE); + pipeline.set_base_time(element.base_time().unwrap()); + + let mut bus_stream = pipeline.bus().unwrap().stream(); + let element_clone = element.downgrade(); + let pipeline_clone = pipeline.downgrade(); + let session_id_clone = session_id.to_owned(); + + task::spawn(async move { + while let Some(msg) = bus_stream.next().await { + if let Some(element) = element_clone.upgrade() { + let this = Self::from_instance(&element); + match msg.view() { + gst::MessageView::Error(err) => { + gst::error!( + CAT, + "session {} error: {}, details: {:?}", + session_id_clone, + err.error(), + err.debug() + ); + let _ = this.remove_session(&element, &session_id_clone, true); + } + gst::MessageView::StateChanged(state_changed) => { + if let Some(pipeline) = pipeline_clone.upgrade() { + if Some(pipeline.clone().upcast()) == state_changed.src() { + pipeline.debug_to_dot_file_with_ts( + gst::DebugGraphDetails::all(), + format!( + "webrtcsink-session-{}-{:?}-to-{:?}", + session_id_clone, + state_changed.old(), + state_changed.current() + ), + ); + } + } + } + gst::MessageView::Latency(..) => { + if let Some(pipeline) = pipeline_clone.upgrade() { + gst::info!(CAT, obj: &pipeline, "Recalculating latency"); + let _ = pipeline.recalculate_latency(); + } + } + gst::MessageView::Eos(..) => { + gst::error!( + CAT, + "Unexpected end of stream in session {}", + session_id_clone, + ); + let _ = this.remove_session(&element, &session_id_clone, true); + } + _ => (), + } + } + } + }); + + pipeline.set_state(gst::State::Ready).map_err(|err| { + WebRTCSinkError::SessionPipelineError { + session_id: session_id.to_string(), + peer_id: peer_id.to_string(), + details: err.to_string(), + } + })?; + + if settings.enable_data_channel_navigation { + state.navigation_handler = Some(NavigationEventHandler::new(element, &webrtcbin)); + } + + state.sessions.insert(session_id.to_string(), session); + + drop(state); + drop(settings); + + // This is intentionally emitted with the pipeline in the Ready state, + // so that application code can create data channels at the correct + // moment. + element.emit_by_name::<()>("consumer-added", &[&peer_id, &webrtcbin]); + + // We don't connect to on-negotiation-needed, this in order to call the above + // signal without holding the state lock: + // + // Going to Ready triggers synchronous emission of the on-negotiation-needed + // signal, during which time the application may add a data channel, causing + // renegotiation, which we do not support at this time. + // + // This is completely safe, as we know that by now all conditions are gathered: + // webrtcbin is in the Ready state, and all its transceivers have codec_preferences. + self.negotiate(element, &session_id); + + pipeline.set_state(gst::State::Playing).map_err(|err| { + WebRTCSinkError::SessionPipelineError { + session_id: session_id.to_string(), + peer_id: peer_id.to_string(), + details: err.to_string(), + } + })?; + + Ok(()) + } + + /// Called by the signaller to remove a consumer + pub fn remove_session( + &self, + element: &super::WebRTCSink, + session_id: &str, + signal: bool, + ) -> Result<(), WebRTCSinkError> { + let mut state = self.state.lock().unwrap(); + + if !state.sessions.contains_key(session_id) { + return Err(WebRTCSinkError::NoSessionWithId(session_id.to_string())); + } + + if let Some(session) = state.end_session(element, session_id, signal) { + drop(state); + element.emit_by_name::<()>("consumer-removed", &[&session.peer_id, &session.webrtcbin]); + } + + Ok(()) + } + + fn process_loss_stats( + &self, + element: &super::WebRTCSink, + session_id: &str, + stats: &gst::Structure, + ) { + let mut state = element.imp().state.lock().unwrap(); + if let Some(mut session) = state.sessions.get_mut(session_id) { + if let Some(congestion_controller) = session.congestion_controller.as_mut() { + congestion_controller.loss_control(&element, stats, &mut session.encoders); + } + session.stats = stats.to_owned(); + } + } + + fn process_stats( + &self, + element: &super::WebRTCSink, + webrtcbin: gst::Element, + session_id: &str, + ) { + let session_id = session_id.to_string(); + let promise = gst::Promise::with_change_func( + glib::clone!(@strong session_id, @weak element => move |reply| { + if let Ok(Some(stats)) = reply { + + let mut state = element.imp().state.lock().unwrap(); + if let Some(mut session) = state.sessions.get_mut(&session_id) { + if let Some(congestion_controller) = session.congestion_controller.as_mut() { + congestion_controller.delay_control(&element, stats, &mut session.encoders,); + } + session.stats = stats.to_owned(); + } + } + }), + ); + + webrtcbin.emit_by_name::<()>("get-stats", &[&None::<gst::Pad>, &promise]); + } + + fn set_rtptrxsend(&self, element: &super::WebRTCSink, peer_id: &str, rtprtxsend: gst::Element) { + let mut state = element.imp().state.lock().unwrap(); + + if let Some(session) = state.sessions.get_mut(peer_id) { + session.rtprtxsend = Some(rtprtxsend); + } + } + + fn set_bitrate(&self, element: &super::WebRTCSink, peer_id: &str, bitrate: u32) { + let settings = element.imp().settings.lock().unwrap(); + let mut state = element.imp().state.lock().unwrap(); + + if let Some(session) = state.sessions.get_mut(peer_id) { + let fec_ratio = { + if settings.do_fec && bitrate > DO_FEC_THRESHOLD { + (bitrate as f64 - DO_FEC_THRESHOLD as f64) + / (session.cc_info.max_bitrate as f64 - DO_FEC_THRESHOLD as f64) + } else { + 0f64 + } + }; + + let fec_percentage = fec_ratio * 50f64; + let encoders_bitrate = ((bitrate as f64) + / (1. + (fec_percentage / 100.)) + / (session.encoders.len() as f64)) as i32; + + if let Some(ref rtpxsend) = session.rtprtxsend.as_ref() { + rtpxsend.set_property("stuffing-kbps", (bitrate as f64 / 1000.) as i32); + } + + for encoder in session.encoders.iter_mut() { + encoder.set_bitrate(element, encoders_bitrate); + encoder + .transceiver + .set_property("fec-percentage", fec_percentage as u32); + } + } + } + + fn on_remote_description_set(&self, element: &super::WebRTCSink, session_id: String) { + let mut state = self.state.lock().unwrap(); + let mut remove = false; + let codecs = state.codecs.clone(); + + if let Some(mut session) = state.sessions.remove(&session_id) { + for webrtc_pad in session.webrtc_pads.clone().values() { + let transceiver = webrtc_pad + .pad + .property::<gst_webrtc::WebRTCRTPTransceiver>("transceiver"); + + if let Some(mid) = transceiver.mid() { + state + .mids + .insert(mid.to_string(), webrtc_pad.stream_name.clone()); + } + + if let Some(producer) = state + .streams + .get(&webrtc_pad.stream_name) + .and_then(|stream| stream.producer.clone()) + { + drop(state); + if let Err(err) = + session.connect_input_stream(element, &producer, webrtc_pad, &codecs) + { + gst::error!( + CAT, + obj: element, + "Failed to connect input stream {} for session {}: {}", + webrtc_pad.stream_name, + session_id, + err + ); + remove = true; + state = self.state.lock().unwrap(); + break; + } + state = self.state.lock().unwrap(); + } else { + gst::error!( + CAT, + obj: element, + "No producer to connect session {} to", + session_id, + ); + remove = true; + break; + } + } + + session.pipeline.debug_to_dot_file_with_ts( + gst::DebugGraphDetails::all(), + format!("webrtcsink-peer-{}-remote-description-set", session_id,), + ); + + let element_clone = element.downgrade(); + let webrtcbin = session.webrtcbin.downgrade(); + task::spawn(async move { + let mut interval = + async_std::stream::interval(std::time::Duration::from_millis(100)); + + while interval.next().await.is_some() { + let element_clone = element_clone.clone(); + if let (Some(webrtcbin), Some(element)) = + (webrtcbin.upgrade(), element_clone.upgrade()) + { + element + .imp() + .process_stats(&element, webrtcbin, &session_id); + } else { + break; + } + } + }); + + if remove { + state.finalize_session(element, &mut session, true); + } else { + state.sessions.insert(session.id.clone(), session); + } + } + } + + /// Called by the signaller with an ice candidate + pub fn handle_ice( + &self, + _element: &super::WebRTCSink, + session_id: &str, + sdp_m_line_index: Option<u32>, + _sdp_mid: Option<String>, + candidate: &str, + ) -> Result<(), WebRTCSinkError> { + let state = self.state.lock().unwrap(); + + let sdp_m_line_index = sdp_m_line_index.ok_or(WebRTCSinkError::MandatorySdpMlineIndex)?; + + if let Some(session) = state.sessions.get(session_id) { + gst::trace!(CAT, "adding ice candidate for session {}", session_id); + session + .webrtcbin + .emit_by_name::<()>("add-ice-candidate", &[&sdp_m_line_index, &candidate]); + Ok(()) + } else { + Err(WebRTCSinkError::NoSessionWithId(session_id.to_string())) + } + } + + /// Called by the signaller with an answer to our offer + pub fn handle_sdp( + &self, + element: &super::WebRTCSink, + session_id: &str, + desc: &gst_webrtc::WebRTCSessionDescription, + ) -> Result<(), WebRTCSinkError> { + let mut state = self.state.lock().unwrap(); + + if let Some(session) = state.sessions.get_mut(session_id) { + let sdp = desc.sdp(); + + session.sdp = Some(sdp.to_owned()); + + for webrtc_pad in session.webrtc_pads.values_mut() { + let media_idx = webrtc_pad.media_idx; + /* TODO: support partial answer, webrtcbin doesn't seem + * very well equipped to deal with this at the moment */ + if let Some(media) = sdp.media(media_idx) { + if media.attribute_val("inactive").is_some() { + let media_str = sdp + .media(webrtc_pad.media_idx) + .and_then(|media| media.as_text().ok()); + + gst::warning!( + CAT, + "consumer from session {} refused media {}: {:?}", + session_id, + media_idx, + media_str + ); + state.end_session(element, session_id, true); + + return Err(WebRTCSinkError::ConsumerRefusedMedia { + session_id: session_id.to_string(), + media_idx, + }); + } + } + + if let Some(payload) = sdp + .media(webrtc_pad.media_idx) + .and_then(|media| media.format(0)) + .and_then(|format| format.parse::<i32>().ok()) + { + webrtc_pad.payload = Some(payload); + } else { + gst::warning!( + CAT, + "consumer from session {} did not provide valid payload for media index {} for session {}", + session_id, + media_idx, + session_id, + ); + + state.end_session(element, session_id, true); + + return Err(WebRTCSinkError::ConsumerNoValidPayload { + session_id: session_id.to_string(), + media_idx, + }); + } + } + + let element = element.downgrade(); + let session_id = session_id.to_string(); + + let promise = gst::Promise::with_change_func(move |reply| { + gst::debug!(CAT, "received reply {:?}", reply); + if let Some(element) = element.upgrade() { + let this = Self::from_instance(&element); + + this.on_remote_description_set(&element, session_id); + } + }); + + session + .webrtcbin + .emit_by_name::<()>("set-remote-description", &[desc, &promise]); + + Ok(()) + } else { + Err(WebRTCSinkError::NoSessionWithId(session_id.to_string())) + } + } + + async fn run_discovery_pipeline( + _element: &super::WebRTCSink, + codec: &Codec, + caps: &gst::Caps, + ) -> Result<gst::Structure, Error> { + let pipe = PipelineWrapper(gst::Pipeline::new(None)); + + let src = if codec.is_video() { + make_element("videotestsrc", None)? + } else { + make_element("audiotestsrc", None)? + }; + let mut elements = vec![src.clone()]; + + if codec.is_video() { + elements.push(make_converter_for_video_caps(caps)?); + } + + let capsfilter = make_element("capsfilter", None)?; + elements.push(capsfilter.clone()); + let elements_slice = &elements.iter().collect::<Vec<_>>(); + pipe.0.add_many(elements_slice).unwrap(); + gst::Element::link_many(elements_slice) + .with_context(|| format!("Running discovery pipeline for caps {}", caps))?; + + let (_, _, pay) = setup_encoding(&pipe.0, &capsfilter, caps, codec, None, true)?; + + let sink = make_element("fakesink", None)?; + + pipe.0.add(&sink).unwrap(); + + pay.link(&sink) + .with_context(|| format!("Running discovery pipeline for caps {}", caps))?; + + capsfilter.set_property("caps", caps); + + src.set_property("num-buffers", 1); + + let mut stream = pipe.0.bus().unwrap().stream(); + + pipe.0 + .set_state(gst::State::Playing) + .with_context(|| format!("Running discovery pipeline for caps {}", caps))?; + + while let Some(msg) = stream.next().await { + match msg.view() { + gst::MessageView::Error(err) => { + pipe.0.debug_to_dot_file_with_ts( + gst::DebugGraphDetails::all(), + "webrtcsink-discovery-error", + ); + return Err(err.error().into()); + } + gst::MessageView::Eos(_) => { + let caps = pay.static_pad("src").unwrap().current_caps().unwrap(); + + pipe.0.debug_to_dot_file_with_ts( + gst::DebugGraphDetails::all(), + "webrtcsink-discovery-done", + ); + + if let Some(s) = caps.structure(0) { + let mut s = s.to_owned(); + s.remove_fields(&[ + "timestamp-offset", + "seqnum-offset", + "ssrc", + "sprop-parameter-sets", + "a-framerate", + ]); + s.set("payload", codec.payload); + return Ok(s); + } else { + return Err(anyhow!("Discovered empty caps")); + } + } + _ => { + continue; + } + } + } + + unreachable!() + } + + async fn lookup_caps( + element: &super::WebRTCSink, + name: String, + in_caps: gst::Caps, + codecs: &BTreeMap<i32, Codec>, + ) -> (String, gst::Caps) { + let sink_caps = in_caps.as_ref().to_owned(); + + let is_video = match sink_caps.structure(0).unwrap().name() { + "video/x-raw" => true, + "audio/x-raw" => false, + _ => unreachable!(), + }; + + let mut payloader_caps = gst::Caps::new_empty(); + let payloader_caps_mut = payloader_caps.make_mut(); + + let futs = codecs + .iter() + .filter(|(_, codec)| codec.is_video() == is_video) + .map(|(_, codec)| WebRTCSink::run_discovery_pipeline(element, codec, &sink_caps)); + + for ret in futures::future::join_all(futs).await { + match ret { + Ok(s) => { + payloader_caps_mut.append_structure(s); + } + Err(err) => { + /* We don't consider this fatal, as long as we end up with one + * potential codec for each input stream + */ + gst::warning!( + CAT, + obj: element, + "Codec discovery pipeline failed: {}", + err + ); + } + } + } + + (name, payloader_caps) + } + + async fn lookup_streams_caps(&self, element: &super::WebRTCSink) -> Result<(), Error> { + let codecs = self.lookup_codecs(); + let futs: Vec<_> = self + .state + .lock() + .unwrap() + .streams + .iter() + .map(|(name, stream)| { + WebRTCSink::lookup_caps( + element, + name.to_owned(), + stream.in_caps.as_ref().unwrap().to_owned(), + &codecs, + ) + }) + .collect(); + + let caps: Vec<(String, gst::Caps)> = futures::future::join_all(futs).await; + + let mut state = self.state.lock().unwrap(); + + for (name, caps) in caps { + if caps.is_empty() { + return Err(anyhow!("No caps found for stream {}", name)); + } + + if let Some(mut stream) = state.streams.get_mut(&name) { + stream.out_caps = Some(caps); + } + } + + state.codecs = codecs; + + Ok(()) + } + + fn gather_stats(&self) -> gst::Structure { + gst::Structure::from_iter( + "application/x-webrtcsink-stats", + self.state + .lock() + .unwrap() + .sessions + .iter() + .map(|(name, consumer)| (name.as_str(), consumer.gather_stats().to_send_value())), + ) + } + + fn sink_event(&self, pad: &gst::Pad, element: &super::WebRTCSink, event: gst::Event) -> bool { + use gst::EventView; + + match event.view() { + EventView::Caps(e) => { + if let Some(caps) = pad.current_caps() { + if caps.is_strictly_equal(e.caps()) { + // Nothing changed + true + } else { + gst::error!(CAT, obj: pad, "Renegotiation is not supported"); + false + } + } else { + gst::info!(CAT, obj: pad, "Received caps event {:?}", e); + + let mut all_pads_have_caps = true; + + self.state + .lock() + .unwrap() + .streams + .iter_mut() + .for_each(|(_, mut stream)| { + if stream.sink_pad.upcast_ref::<gst::Pad>() == pad { + stream.in_caps = Some(e.caps().to_owned()); + } else if stream.in_caps.is_none() { + all_pads_have_caps = false; + } + }); + + if all_pads_have_caps { + let element_clone = element.downgrade(); + task::spawn(async move { + if let Some(element) = element_clone.upgrade() { + let this = Self::from_instance(&element); + let (fut, handle) = + futures::future::abortable(this.lookup_streams_caps(&element)); + + let (codecs_done_sender, codecs_done_receiver) = + futures::channel::oneshot::channel(); + + // Compiler isn't budged by dropping state before await, + // so let's make a new scope instead. + { + let mut state = this.state.lock().unwrap(); + state.codecs_abort_handle = Some(handle); + state.codecs_done_receiver = Some(codecs_done_receiver); + } + + match fut.await { + Ok(Err(err)) => { + gst::error!(CAT, obj: &element, "error: {}", err); + gst::element_error!( + element, + gst::StreamError::CodecNotFound, + ["Failed to look up output caps: {}", err] + ); + } + Ok(Ok(_)) => { + let mut state = this.state.lock().unwrap(); + state.codec_discovery_done = true; + state.maybe_start_signaller(&element); + } + _ => (), + } + + let _ = codecs_done_sender.send(()); + } + }); + } + + gst::Pad::event_default(pad, Some(element), event) + } + } + _ => gst::Pad::event_default(pad, Some(element), event), + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for WebRTCSink { + const NAME: &'static str = "RsWebRTCSink"; + type Type = super::WebRTCSink; + type ParentType = gst::Bin; + type Interfaces = (gst::ChildProxy, gst_video::Navigation); +} + +impl ObjectImpl for WebRTCSink { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { + vec![ + glib::ParamSpecBoxed::new( + "video-caps", + "Video encoder caps", + "Governs what video codecs will be proposed", + gst::Caps::static_type(), + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecBoxed::new( + "audio-caps", + "Audio encoder caps", + "Governs what audio codecs will be proposed", + gst::Caps::static_type(), + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecString::new( + "stun-server", + "STUN Server", + "The STUN server of the form stun://hostname:port", + DEFAULT_STUN_SERVER, + glib::ParamFlags::READWRITE, + ), + glib::ParamSpecString::new( + "turn-server", + "TURN Server", + "The TURN server of the form turn(s)://username:password@host:port.", + None, + glib::ParamFlags::READWRITE, + ), + glib::ParamSpecEnum::new( + "congestion-control", + "Congestion control", + "Defines how congestion is controlled, if at all", + WebRTCSinkCongestionControl::static_type(), + DEFAULT_CONGESTION_CONTROL as i32, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecUInt::new( + "min-bitrate", + "Minimal Bitrate", + "Minimal bitrate to use (in bit/sec) when computing it through the congestion control algorithm", + 1, + u32::MAX as u32, + DEFAULT_MIN_BITRATE, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecUInt::new( + "max-bitrate", + "Minimal Bitrate", + "Minimal bitrate to use (in bit/sec) when computing it through the congestion control algorithm", + 1, + u32::MAX as u32, + DEFAULT_MAX_BITRATE, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecUInt::new( + "start-bitrate", + "Start Bitrate", + "Start bitrate to use (in bit/sec)", + 1, + u32::MAX as u32, + DEFAULT_START_BITRATE, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + ), + glib::ParamSpecBoxed::new( + "stats", + "Consumer statistics", + "Statistics for the current consumers", + gst::Structure::static_type(), + glib::ParamFlags::READABLE, + ), + glib::ParamSpecBoolean::new( + "do-fec", + "Do Forward Error Correction", + "Whether the element should negotiate and send FEC data", + DEFAULT_DO_FEC, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY + ), + glib::ParamSpecBoolean::new( + "do-retransmission", + "Do retransmission", + "Whether the element should offer to honor retransmission requests", + DEFAULT_DO_RETRANSMISSION, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY + ), + glib::ParamSpecBoolean::new( + "enable-data-channel-navigation", + "Enable data channel navigation", + "Enable navigation events through a dedicated WebRTCDataChannel", + DEFAULT_ENABLE_DATA_CHANNEL_NAVIGATION, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY + ), + glib::ParamSpecBoxed::new( + "meta", + "Meta", + "Free form metadata about the producer", + gst::Structure::static_type(), + glib::ParamFlags::READWRITE, + ), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property( + &self, + _id: usize, + value: &glib::Value, + pspec: &glib::ParamSpec, + ) { + match pspec.name() { + "video-caps" => { + let mut settings = self.settings.lock().unwrap(); + settings.video_caps = value + .get::<Option<gst::Caps>>() + .expect("type checked upstream") + .unwrap_or_else(gst::Caps::new_empty); + } + "audio-caps" => { + let mut settings = self.settings.lock().unwrap(); + settings.audio_caps = value + .get::<Option<gst::Caps>>() + .expect("type checked upstream") + .unwrap_or_else(gst::Caps::new_empty); + } + "stun-server" => { + let mut settings = self.settings.lock().unwrap(); + settings.stun_server = value + .get::<Option<String>>() + .expect("type checked upstream") + } + "turn-server" => { + let mut settings = self.settings.lock().unwrap(); + settings.turn_server = value + .get::<Option<String>>() + .expect("type checked upstream") + } + "congestion-control" => { + let mut settings = self.settings.lock().unwrap(); + settings.cc_info.heuristic = value + .get::<WebRTCSinkCongestionControl>() + .expect("type checked upstream"); + } + "min-bitrate" => { + let mut settings = self.settings.lock().unwrap(); + settings.cc_info.min_bitrate = value.get::<u32>().expect("type checked upstream"); + } + "max-bitrate" => { + let mut settings = self.settings.lock().unwrap(); + settings.cc_info.max_bitrate = value.get::<u32>().expect("type checked upstream"); + } + "start-bitrate" => { + let mut settings = self.settings.lock().unwrap(); + settings.cc_info.start_bitrate = value.get::<u32>().expect("type checked upstream"); + } + "do-fec" => { + let mut settings = self.settings.lock().unwrap(); + settings.do_fec = value.get::<bool>().expect("type checked upstream"); + } + "do-retransmission" => { + let mut settings = self.settings.lock().unwrap(); + settings.do_retransmission = value.get::<bool>().expect("type checked upstream"); + } + "enable-data-channel-navigation" => { + let mut settings = self.settings.lock().unwrap(); + settings.enable_data_channel_navigation = + value.get::<bool>().expect("type checked upstream"); + } + "meta" => { + let mut settings = self.settings.lock().unwrap(); + settings.meta = value + .get::<Option<gst::Structure>>() + .expect("type checked upstream") + } + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "video-caps" => { + let settings = self.settings.lock().unwrap(); + settings.video_caps.to_value() + } + "audio-caps" => { + let settings = self.settings.lock().unwrap(); + settings.audio_caps.to_value() + } + "congestion-control" => { + let settings = self.settings.lock().unwrap(); + settings.cc_info.heuristic.to_value() + } + "stun-server" => { + let settings = self.settings.lock().unwrap(); + settings.stun_server.to_value() + } + "turn-server" => { + let settings = self.settings.lock().unwrap(); + settings.turn_server.to_value() + } + "min-bitrate" => { + let settings = self.settings.lock().unwrap(); + settings.cc_info.min_bitrate.to_value() + } + "max-bitrate" => { + let settings = self.settings.lock().unwrap(); + settings.cc_info.max_bitrate.to_value() + } + "start-bitrate" => { + let settings = self.settings.lock().unwrap(); + settings.cc_info.start_bitrate.to_value() + } + "do-fec" => { + let settings = self.settings.lock().unwrap(); + settings.do_fec.to_value() + } + "do-retransmission" => { + let settings = self.settings.lock().unwrap(); + settings.do_retransmission.to_value() + } + "enable-data-channel-navigation" => { + let settings = self.settings.lock().unwrap(); + settings.enable_data_channel_navigation.to_value() + } + "stats" => self.gather_stats().to_value(), + "meta" => { + let settings = self.settings.lock().unwrap(); + settings.meta.to_value() + } + _ => unimplemented!(), + } + } + + fn signals() -> &'static [glib::subclass::Signal] { + static SIGNALS: Lazy<Vec<glib::subclass::Signal>> = Lazy::new(|| { + vec![ + /* + * RsWebRTCSink::consumer-added: + * @consumer_id: Identifier of the consumer added + * @webrtcbin: The new webrtcbin + * + * This signal can be used to tweak @webrtcbin, creating a data + * channel for example. + */ + glib::subclass::Signal::builder("consumer-added") + .param_types([String::static_type(), gst::Element::static_type()]) + .build(), + /* + * RsWebRTCSink::consumer_removed: + * @consumer_id: Identifier of the consumer that was removed + * @webrtcbin: The webrtcbin connected to the newly removed consumer + * + * This signal is emitted right after the connection with a consumer + * has been dropped. + */ + glib::subclass::Signal::builder("consumer-removed") + .param_types([String::static_type(), gst::Element::static_type()]) + .build(), + /* + * RsWebRTCSink::get_sessions: + * + * List all sessions (by ID). + */ + glib::subclass::Signal::builder("get-sessions") + .action() + .class_handler(|_, args| { + let element = args[0].get::<super::WebRTCSink>().expect("signal arg"); + let this = element.imp(); + + let res = Some( + this.state + .lock() + .unwrap() + .sessions + .keys() + .cloned() + .collect::<Vec<String>>() + .to_value(), + ); + res + }) + .return_type::<Vec<String>>() + .build(), + /* + * RsWebRTCSink::encoder-setup: + * @consumer_id: Identifier of the consumer + * @pad_name: The name of the corresponding input pad + * @encoder: The constructed encoder + * + * This signal can be used to tweak @encoder properties. + * + * Returns: True if the encoder is entirely configured, + * False to let other handlers run + */ + glib::subclass::Signal::builder("encoder-setup") + .param_types([ + String::static_type(), + String::static_type(), + gst::Element::static_type(), + ]) + .return_type::<bool>() + .accumulator(|_hint, _ret, value| !value.get::<bool>().unwrap()) + .class_handler(|_, args| { + let element = args[0].get::<super::WebRTCSink>().expect("signal arg"); + let enc = args[3].get::<gst::Element>().unwrap(); + + gst::debug!( + CAT, + obj: &element, + "applying default configuration on encoder {:?}", + enc + ); + + let this = element.imp(); + let settings = this.settings.lock().unwrap(); + configure_encoder(&enc, settings.cc_info.start_bitrate); + + // Return false here so that latter handlers get called + Some(false.to_value()) + }) + .build(), + ] + }); + + SIGNALS.as_ref() + } + + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.instance(); + obj.set_suppressed_flags(gst::ElementFlags::SINK | gst::ElementFlags::SOURCE); + obj.set_element_flags(gst::ElementFlags::SINK); + } +} + +impl GstObjectImpl for WebRTCSink {} + +impl ElementImpl for WebRTCSink { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "WebRTCSink", + "Sink/Network/WebRTC", + "WebRTC sink", + "Mathieu Duponchelle <mathieu@centricular.com>", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| { + let caps = gst::Caps::builder_full() + .structure(gst::Structure::builder("video/x-raw").build()) + .structure_with_features( + gst::Structure::builder("video/x-raw").build(), + gst::CapsFeatures::new(&[CUDA_MEMORY_FEATURE]), + ) + .structure_with_features( + gst::Structure::builder("video/x-raw").build(), + gst::CapsFeatures::new(&[GL_MEMORY_FEATURE]), + ) + .structure_with_features( + gst::Structure::builder("video/x-raw").build(), + gst::CapsFeatures::new(&[NVMM_MEMORY_FEATURE]), + ) + .build(); + let video_pad_template = gst::PadTemplate::new( + "video_%u", + gst::PadDirection::Sink, + gst::PadPresence::Request, + &caps, + ) + .unwrap(); + + let caps = gst::Caps::builder("audio/x-raw").build(); + let audio_pad_template = gst::PadTemplate::new( + "audio_%u", + gst::PadDirection::Sink, + gst::PadPresence::Request, + &caps, + ) + .unwrap(); + + vec![video_pad_template, audio_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } + + fn request_new_pad( + &self, + templ: &gst::PadTemplate, + _name: Option<String>, + _caps: Option<&gst::Caps>, + ) -> Option<gst::Pad> { + let element = self.instance(); + if element.current_state() > gst::State::Ready { + gst::error!(CAT, "element pads can only be requested before starting"); + return None; + } + + let mut state = self.state.lock().unwrap(); + + let name = if templ.name().starts_with("video_") { + let name = format!("video_{}", state.video_serial); + state.video_serial += 1; + name + } else { + let name = format!("audio_{}", state.audio_serial); + state.audio_serial += 1; + name + }; + + let sink_pad = gst::GhostPad::builder_with_template(templ, Some(name.as_str())) + .event_function(|pad, parent, event| { + WebRTCSink::catch_panic_pad_function( + parent, + || false, + |this| this.sink_event(pad.upcast_ref(), &*this.instance(), event), + ) + }) + .build(); + + sink_pad.set_active(true).unwrap(); + sink_pad.use_fixed_caps(); + element.add_pad(&sink_pad).unwrap(); + + state.streams.insert( + name, + InputStream { + sink_pad: sink_pad.clone(), + producer: None, + in_caps: None, + out_caps: None, + clocksync: None, + }, + ); + + Some(sink_pad.upcast()) + } + + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result<gst::StateChangeSuccess, gst::StateChangeError> { + let element = self.instance(); + if let gst::StateChange::ReadyToPaused = transition { + if let Err(err) = self.prepare(&*element) { + gst::element_error!( + element, + gst::StreamError::Failed, + ["Failed to prepare: {}", err] + ); + return Err(gst::StateChangeError); + } + } + + let mut ret = self.parent_change_state(transition); + + match transition { + gst::StateChange::PausedToReady => { + if let Err(err) = self.unprepare(&*element) { + gst::element_error!( + element, + gst::StreamError::Failed, + ["Failed to unprepare: {}", err] + ); + return Err(gst::StateChangeError); + } + } + gst::StateChange::ReadyToPaused => { + ret = Ok(gst::StateChangeSuccess::NoPreroll); + } + gst::StateChange::PausedToPlaying => { + let mut state = self.state.lock().unwrap(); + state.maybe_start_signaller(&*element); + } + _ => (), + } + + ret + } +} + +impl BinImpl for WebRTCSink {} + +impl ChildProxyImpl for WebRTCSink { + fn child_by_index(&self, _index: u32) -> Option<glib::Object> { + None + } + + fn children_count(&self) -> u32 { + 0 + } + + fn child_by_name(&self, name: &str) -> Option<glib::Object> { + match name { + "signaller" => Some( + self.state + .lock() + .unwrap() + .signaller + .as_ref() + .as_ref() + .clone(), + ), + _ => None, + } + } +} + +impl NavigationImpl for WebRTCSink { + fn send_event(&self, event_def: gst::Structure) { + let mut state = self.state.lock().unwrap(); + let event = gst::event::Navigation::new(event_def); + + state.streams.iter_mut().for_each(|(_, stream)| { + if stream.sink_pad.name().starts_with("video_") { + gst::log!(CAT, "Navigating to: {:?}", event); + // FIXME: Handle multi tracks. + if !stream.sink_pad.push_event(event.clone()) { + gst::info!(CAT, "Could not send event: {:?}", event); + } + } + }); + } +} diff --git a/net/webrtc/plugins/src/webrtcsink/mod.rs b/net/webrtc/plugins/src/webrtcsink/mod.rs new file mode 100644 index 00000000..9d6913c9 --- /dev/null +++ b/net/webrtc/plugins/src/webrtcsink/mod.rs @@ -0,0 +1,164 @@ +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; +use std::error::Error; + +mod homegrown_cc; +mod imp; + +glib::wrapper! { + pub struct WebRTCSink(ObjectSubclass<imp::WebRTCSink>) @extends gst::Bin, gst::Element, gst::Object, @implements gst::ChildProxy, gst_video::Navigation; +} + +unsafe impl Send for WebRTCSink {} +unsafe impl Sync for WebRTCSink {} + +#[derive(thiserror::Error, Debug)] +pub enum WebRTCSinkError { + #[error("no session with id")] + NoSessionWithId(String), + #[error("consumer refused media")] + ConsumerRefusedMedia { session_id: String, media_idx: u32 }, + #[error("consumer did not provide valid payload for media")] + ConsumerNoValidPayload { session_id: String, media_idx: u32 }, + #[error("SDP mline index is currently mandatory")] + MandatorySdpMlineIndex, + #[error("duplicate session id")] + DuplicateSessionId(String), + #[error("error setting up consumer pipeline")] + SessionPipelineError { + session_id: String, + peer_id: String, + details: String, + }, +} + +pub trait Signallable: Sync + Send + 'static { + fn start(&mut self, element: &WebRTCSink) -> Result<(), Box<dyn Error>>; + + fn handle_sdp( + &mut self, + element: &WebRTCSink, + session_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) -> Result<(), Box<dyn Error>>; + + /// sdp_mid is exposed for future proofing, see + /// https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1174, + /// at the moment sdp_m_line_index will always be Some and sdp_mid will always + /// be None + fn handle_ice( + &mut self, + element: &WebRTCSink, + session_id: &str, + candidate: &str, + sdp_m_line_index: Option<u32>, + sdp_mid: Option<String>, + ) -> Result<(), Box<dyn Error>>; + + fn session_ended(&mut self, element: &WebRTCSink, session_id: &str); + + fn stop(&mut self, element: &WebRTCSink); +} + +/// When providing a signaller, we expect it to both be a GObject +/// and be Signallable. This is arguably a bit strange, but exposing +/// a GInterface from rust is at the moment a bit awkward, so I went +/// for a rust interface for now. The reason the signaller needs to be +/// a GObject is to make its properties available through the GstChildProxy +/// interface. +pub trait SignallableObject: AsRef<glib::Object> + Signallable {} + +impl<T: AsRef<glib::Object> + Signallable> SignallableObject for T {} + +impl Default for WebRTCSink { + fn default() -> Self { + glib::Object::new::<Self>(&[]) + } +} + +impl WebRTCSink { + pub fn with_signaller(signaller: Box<dyn SignallableObject>) -> Self { + let ret: WebRTCSink = glib::Object::new(&[]); + + let ws = imp::WebRTCSink::from_instance(&ret); + + ws.set_signaller(signaller).unwrap(); + + ret + } + + pub fn handle_sdp( + &self, + session_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) -> Result<(), WebRTCSinkError> { + let ws = imp::WebRTCSink::from_instance(self); + + ws.handle_sdp(self, session_id, sdp) + } + + /// sdp_mid is exposed for future proofing, see + /// https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1174, + /// at the moment sdp_m_line_index must be Some + pub fn handle_ice( + &self, + session_id: &str, + sdp_m_line_index: Option<u32>, + sdp_mid: Option<String>, + candidate: &str, + ) -> Result<(), WebRTCSinkError> { + let ws = imp::WebRTCSink::from_instance(self); + + ws.handle_ice(self, session_id, sdp_m_line_index, sdp_mid, candidate) + } + + pub fn handle_signalling_error(&self, error: Box<dyn Error + Send + Sync>) { + let ws = imp::WebRTCSink::from_instance(self); + + ws.handle_signalling_error(self, anyhow::anyhow!(error)); + } + + pub fn start_session(&self, session_id: &str, peer_id: &str) -> Result<(), WebRTCSinkError> { + let ws = imp::WebRTCSink::from_instance(self); + + ws.start_session(self, session_id, peer_id) + } + + pub fn end_session(&self, session_id: &str) -> Result<(), WebRTCSinkError> { + let ws = imp::WebRTCSink::from_instance(self); + + ws.remove_session(self, session_id, false) + } +} + +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum)] +#[repr(u32)] +#[enum_type(name = "GstWebRTCSinkCongestionControl")] +pub enum WebRTCSinkCongestionControl { + #[enum_value(name = "Disabled: no congestion control is applied", nick = "disabled")] + Disabled, + #[enum_value(name = "Homegrown: simple sender-side heuristic", nick = "homegrown")] + Homegrown, + #[enum_value(name = "Google Congestion Control algorithm", nick = "gcc")] + GoogleCongestionControl, +} + +#[glib::flags(name = "GstWebRTCSinkMitigationMode")] +enum WebRTCSinkMitigationMode { + #[flags_value(name = "No mitigation applied", nick = "none")] + NONE = 0b00000000, + #[flags_value(name = "Lowered resolution", nick = "downscaled")] + DOWNSCALED = 0b00000001, + #[flags_value(name = "Lowered framerate", nick = "downsampled")] + DOWNSAMPLED = 0b00000010, +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "webrtcsink", + gst::Rank::None, + WebRTCSink::static_type(), + ) +} diff --git a/net/webrtc/protocol/Cargo.toml b/net/webrtc/protocol/Cargo.toml new file mode 100644 index 00000000..e3e3099c --- /dev/null +++ b/net/webrtc/protocol/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name="webrtcsink-protocol" +version = "0.1.0" +edition = "2018" +authors = ["Mathieu Duponchelle <mathieu@centricular.com>"] +license = "MPL-2.0" +description = "GStreamer WebRTC sink default protocol" +repository = "https://github.com/centricular/webrtcsink/" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/net/webrtc/protocol/src/lib.rs b/net/webrtc/protocol/src/lib.rs new file mode 100644 index 00000000..9af9e929 --- /dev/null +++ b/net/webrtc/protocol/src/lib.rs @@ -0,0 +1,144 @@ +/// The default protocol used by the signalling server +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Peer { + pub id: String, + #[serde(default)] + pub meta: Option<serde_json::Value>, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[serde(tag = "type")] +#[serde(rename_all = "camelCase")] +/// Messages sent from the server to peers +pub enum OutgoingMessage { + /// Welcoming message, sets the Peer ID linked to a new connection + Welcome { peer_id: String }, + /// Notifies listeners that a peer status has changed + PeerStatusChanged(PeerStatus), + /// Instructs a peer to generate an offer and inform about the session ID + #[serde(rename_all = "camelCase")] + StartSession { peer_id: String, session_id: String }, + /// Let consumer know that the requested session is starting with the specified identifier + #[serde(rename_all = "camelCase")] + SessionStarted { peer_id: String, session_id: String }, + /// Signals that the session the peer was in was ended + #[serde(rename_all = "camelCase")] + EndSession(EndSessionMessage), + /// Messages directly forwarded from one peer to another + Peer(PeerMessage), + /// Provides the current list of consumer peers + List { producers: Vec<Peer> }, + /// Notifies that an error occurred with the peer's current session + Error { details: String }, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +#[serde(rename_all = "camelCase")] +/// Register with a peer type +pub enum PeerRole { + /// Register as a producer + #[serde(rename_all = "camelCase")] + Producer, + /// Register as a listener + #[serde(rename_all = "camelCase")] + Listener, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PeerStatus { + pub roles: Vec<PeerRole>, + pub meta: Option<serde_json::Value>, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub peer_id: Option<String>, +} + +impl PeerStatus { + pub fn producing(&self) -> bool { + self.roles.iter().any(|t| matches!(t, PeerRole::Producer)) + } + + pub fn listening(&self) -> bool { + self.roles.iter().any(|t| matches!(t, PeerRole::Listener)) + } +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Ask the server to start a session with a producer peer +pub struct StartSessionMessage { + /// Identifies the peer + pub peer_id: String, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[serde(tag = "type")] +#[serde(rename_all = "camelCase")] +/// Conveys a SDP +pub enum SdpMessage { + /// Conveys an offer + Offer { + /// The SDP + sdp: String, + }, + /// Conveys an answer + Answer { + /// The SDP + sdp: String, + }, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[serde(rename_all = "camelCase")] +/// Contents of the peer message +pub enum PeerMessageInner { + /// Conveys an ICE candidate + #[serde(rename_all = "camelCase")] + Ice { + /// The candidate string + candidate: String, + /// The mline index the candidate applies to + sdp_m_line_index: u32, + }, + Sdp(SdpMessage), +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +/// Messages directly forwarded from one peer to another +pub struct PeerMessage { + pub session_id: String, + #[serde(flatten)] + pub peer_message: PeerMessageInner, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +/// End a session +pub struct EndSessionMessage { + /// The identifier of the session to end + pub session_id: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "type")] +#[serde(rename_all = "camelCase")] +/// Messages received by the server from peers +pub enum IncomingMessage { + /// Internal message to let know about new peers + NewPeer, + /// Set current peer status + SetPeerStatus(PeerStatus), + /// Start a session with a producer peer + StartSession(StartSessionMessage), + /// End an existing session + EndSession(EndSessionMessage), + /// Send a message to a peer the sender is currently in session with + Peer(PeerMessage), + /// Retrieve the current list of producers + List, +} diff --git a/net/webrtc/signalling/Cargo.toml b/net/webrtc/signalling/Cargo.toml new file mode 100644 index 00000000..986c47f3 --- /dev/null +++ b/net/webrtc/signalling/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name="webrtcsink-signalling" +version = "0.1.0" +edition = "2018" +authors = ["Mathieu Duponchelle <mathieu@centricular.com>"] +license = "MPL-2.0" +description = "GStreamer WebRTC sink signalling server" +repository = "https://github.com/centricular/webrtcsink/" + +[dependencies] +anyhow = "1" +async-std = { version = "1", features = ["unstable", "attributes"] } +async-native-tls = "0.4" +async-tungstenite = { version = "0.17", features = ["async-std-runtime", "async-native-tls"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +clap = { version = "4", features = ["derive"] } +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } +tracing-log = "0.1" +futures = "0.3" +uuid = { version = "1", features = ["v4"] } +thiserror = "1" +test-log = { version = "0.2", features = ["trace"], default-features = false } +pin-project-lite = "0.2" +webrtcsink-protocol = { version = "0.1", path="../protocol" } diff --git a/net/webrtc/signalling/src/bin/server.rs b/net/webrtc/signalling/src/bin/server.rs new file mode 100644 index 00000000..65f41055 --- /dev/null +++ b/net/webrtc/signalling/src/bin/server.rs @@ -0,0 +1,101 @@ +use async_std::task; +use clap::Parser; +use tracing_subscriber::prelude::*; +use webrtcsink_signalling::handlers::Handler; +use webrtcsink_signalling::server::Server; + +use anyhow::Error; +use async_native_tls::TlsAcceptor; +use async_std::fs::File as AsyncFile; +use async_std::net::TcpListener; +use tracing::{info, warn}; + +#[derive(Parser, Debug)] +#[clap(about, version, author)] +/// Program arguments +struct Args { + /// Address to listen on + #[clap(short, long, default_value = "0.0.0.0")] + host: String, + /// Port to listen on + #[clap(short, long, default_value_t = 8443)] + port: u16, + /// TLS certificate to use + #[clap(short, long)] + cert: Option<String>, + /// password to TLS certificate + #[clap(long)] + cert_password: Option<String>, +} + +fn initialize_logging(envvar_name: &str) -> Result<(), Error> { + tracing_log::LogTracer::init()?; + let env_filter = tracing_subscriber::EnvFilter::try_from_env(envvar_name) + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let fmt_layer = tracing_subscriber::fmt::layer() + .with_thread_ids(true) + .with_target(true) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ); + let subscriber = tracing_subscriber::Registry::default() + .with(env_filter) + .with(fmt_layer); + tracing::subscriber::set_global_default(subscriber)?; + + Ok(()) +} + +fn main() -> Result<(), Error> { + let args = Args::parse(); + let server = Server::spawn(|stream| Handler::new(stream)); + + initialize_logging("WEBRTCSINK_SIGNALLING_SERVER_LOG")?; + + task::block_on(async move { + let addr = format!("{}:{}", args.host, args.port); + + // Create the event loop and TCP listener we'll accept connections on. + let listener = TcpListener::bind(&addr).await?; + + let acceptor = match args.cert { + Some(cert) => { + let key = AsyncFile::open(cert).await?; + Some(TlsAcceptor::new(key, args.cert_password.as_deref().unwrap_or("")).await?) + } + None => None, + }; + + info!("Listening on: {}", addr); + + while let Ok((stream, _)) = listener.accept().await { + let mut server_clone = server.clone(); + + let address = match stream.peer_addr() { + Ok(address) => address, + Err(err) => { + warn!("Connected peer with no address: {}", err); + continue; + } + }; + + info!("Accepting connection from {}", address); + + if let Some(ref acceptor) = acceptor { + let stream = match acceptor.accept(stream).await { + Ok(stream) => stream, + Err(err) => { + warn!("Failed to accept TLS connection from {}: {}", address, err); + continue; + } + }; + task::spawn(async move { server_clone.accept_async(stream).await }); + } else { + task::spawn(async move { server_clone.accept_async(stream).await }); + } + } + + Ok(()) + }) +} diff --git a/net/webrtc/signalling/src/handlers/mod.rs b/net/webrtc/signalling/src/handlers/mod.rs new file mode 100644 index 00000000..011bf910 --- /dev/null +++ b/net/webrtc/signalling/src/handlers/mod.rs @@ -0,0 +1,1421 @@ +use anyhow::{anyhow, Error}; +use anyhow::{bail, Context}; +use futures::prelude::*; +use futures::ready; +use p::PeerStatus; +use pin_project_lite::pin_project; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::pin::Pin; +use std::task::{Context as TaskContext, Poll}; +use tracing::log::error; +use tracing::{info, instrument, warn}; +use webrtcsink_protocol as p; + +type PeerId = String; + +#[derive(Clone)] +struct Session { + id: String, + producer: PeerId, + consumer: PeerId, +} + +impl Session { + fn other_peer_id(&self, id: &str) -> Result<&str, Error> { + if self.producer == id { + Ok(&self.consumer) + } else if self.consumer == id { + Ok(&self.producer) + } else { + bail!("Peer {id} is not part of {}", self.id) + } + } +} + +pin_project! { + #[must_use = "streams do nothing unless polled"] + pub struct Handler { + #[pin] + stream: Pin<Box<dyn Stream<Item=(String, Option<p::IncomingMessage>)> + Send>>, + items: VecDeque<(String, p::OutgoingMessage)>, + peers: HashMap<PeerId, PeerStatus>, + sessions: HashMap<String, Session>, + consumer_sessions: HashMap<String, HashSet<String>>, + producer_sessions: HashMap<String, HashSet<String>>, + } +} + +impl Handler { + #[instrument(level = "debug", skip(stream))] + /// Create a handler + pub fn new( + stream: Pin<Box<dyn Stream<Item = (String, Option<p::IncomingMessage>)> + Send>>, + ) -> Self { + Self { + stream, + items: VecDeque::new(), + peers: Default::default(), + sessions: Default::default(), + consumer_sessions: Default::default(), + producer_sessions: Default::default(), + } + } + + #[instrument(level = "trace", skip(self))] + fn handle( + mut self: Pin<&mut Self>, + peer_id: &str, + msg: p::IncomingMessage, + ) -> Result<(), Error> { + match msg { + p::IncomingMessage::NewPeer => { + self.peers.insert(peer_id.to_string(), Default::default()); + self.items.push_back(( + peer_id.into(), + p::OutgoingMessage::Welcome { + peer_id: peer_id.to_string(), + }, + )); + + Ok(()) + } + p::IncomingMessage::SetPeerStatus(status) => self.set_peer_status(peer_id, &status), + p::IncomingMessage::StartSession(message) => { + self.start_session(&message.peer_id, peer_id) + } + p::IncomingMessage::Peer(peermsg) => self.handle_peer_message(peer_id, peermsg), + p::IncomingMessage::List => self.list_producers(peer_id), + p::IncomingMessage::EndSession(msg) => self.end_session(peer_id, &msg.session_id), + } + } + + fn handle_peer_message(&mut self, peer_id: &str, peermsg: p::PeerMessage) -> Result<(), Error> { + let session_id = &peermsg.session_id; + let session = self + .sessions + .get(session_id) + .context(format!("Session {} doesn't exist", session_id))? + .clone(); + + if matches!( + peermsg.peer_message, + p::PeerMessageInner::Sdp(p::SdpMessage::Offer { .. }) + ) && peer_id == session.consumer + { + bail!( + r#"cannot forward offer from "{peer_id}" to "{}" as "{peer_id}" is not the producer"#, + session.producer, + ); + } + + self.items.push_back(( + session.other_peer_id(peer_id)?.to_owned(), + p::OutgoingMessage::Peer(p::PeerMessage { + session_id: session_id.to_string(), + peer_message: peermsg.peer_message.clone(), + }), + )); + + Ok(()) + } + + fn stop_producer(&mut self, peer_id: &str) { + if let Some(session_ids) = self.producer_sessions.remove(peer_id) { + for session_id in session_ids { + if let Err(e) = self.end_session(peer_id, &session_id) { + error!("Could not end session {session_id}: {e:?}"); + } + } + } + } + + fn stop_consumer(&mut self, peer_id: &str) { + if let Some(session_ids) = self.consumer_sessions.remove(peer_id) { + for session_id in session_ids { + if let Err(e) = self.end_session(peer_id, &session_id) { + error!("Could not end session {session_id}: {e:?}"); + } + } + } + } + + #[instrument(level = "debug", skip(self))] + /// Remove a peer, this can cause sessions to be ended + fn remove_peer(&mut self, peer_id: &str) { + info!(peer_id = %peer_id, "removing peer"); + let peer_status = match self.peers.remove(peer_id) { + Some(peer_status) => peer_status, + _ => return, + }; + + self.stop_producer(peer_id); + self.stop_consumer(peer_id); + + for (id, p) in self.peers.iter() { + if !p.listening() { + continue; + } + + let message = p::OutgoingMessage::PeerStatusChanged(PeerStatus { + roles: Default::default(), + meta: peer_status.meta.clone(), + peer_id: Some(peer_id.to_string()), + }); + self.items.push_back((id.to_string(), message)); + } + } + + #[instrument(level = "debug", skip(self))] + /// End a session between two peers + fn end_session(&mut self, peer_id: &str, session_id: &str) -> Result<(), Error> { + let session = self + .sessions + .remove(session_id) + .with_context(|| format!("Session {session_id} doesn't exist"))?; + + self.consumer_sessions + .entry(session.consumer.clone()) + .and_modify(|sessions| { + sessions.remove(session_id); + }); + + self.producer_sessions + .entry(session.producer.clone()) + .and_modify(|sessions| { + sessions.remove(session_id); + }); + + self.items.push_back(( + session.other_peer_id(peer_id)?.to_string(), + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.to_string(), + }), + )); + + Ok(()) + } + + /// List producer peers + #[instrument(level = "debug", skip(self))] + fn list_producers(&mut self, peer_id: &str) -> Result<(), Error> { + self.items.push_back(( + peer_id.to_string(), + p::OutgoingMessage::List { + producers: self + .peers + .iter() + .filter_map(|(peer_id, peer)| { + peer.producing().then_some(p::Peer { + id: peer_id.clone(), + meta: peer.meta.clone(), + }) + }) + .collect(), + }, + )); + + Ok(()) + } + + /// Register peer as a producer + #[instrument(level = "debug", skip(self))] + fn set_peer_status(&mut self, peer_id: &str, status: &p::PeerStatus) -> Result<(), Error> { + let old_status = self + .peers + .get(peer_id) + .context(anyhow!("Peer '{peer_id}' hasn't been welcomed"))?; + + if status == old_status { + info!("Status for '{}' hasn't changed", peer_id); + + return Ok(()); + } + + if old_status.producing() && !status.producing() { + self.stop_producer(peer_id); + } + + let mut status = status.clone(); + status.peer_id = Some(peer_id.to_string()); + self.peers.insert(peer_id.to_string(), status.clone()); + for (id, peer) in &self.peers { + if !peer.listening() { + continue; + } + + self.items.push_back(( + id.to_string(), + p::OutgoingMessage::PeerStatusChanged(p::PeerStatus { + peer_id: Some(peer_id.to_string()), + roles: status.roles.clone(), + meta: status.meta.clone(), + }), + )); + } + + info!(peer_id = %peer_id, "registered as a producer"); + + Ok(()) + } + + /// Start a session between two peers + #[instrument(level = "debug", skip(self))] + fn start_session(&mut self, producer_id: &str, consumer_id: &str) -> Result<(), Error> { + self.peers.get(producer_id).map_or_else( + || Err(anyhow!("Peer '{producer_id}' hasn't been welcomed")), + |peer| { + if !peer.producing() { + Err(anyhow!( + "Peer with id {} is not registered as a producer", + producer_id + )) + } else { + Ok(peer) + } + }, + )?; + + self.peers.get(consumer_id).map_or_else( + || Err(anyhow!("Peer '{consumer_id}' hasn't been welcomed")), + Ok, + )?; + + let session_id = uuid::Uuid::new_v4().to_string(); + self.sessions.insert( + session_id.clone(), + Session { + id: session_id.clone(), + consumer: consumer_id.to_string(), + producer: producer_id.to_string(), + }, + ); + self.consumer_sessions + .entry(consumer_id.to_string()) + .or_insert(HashSet::new()) + .insert(session_id.clone()); + self.producer_sessions + .entry(producer_id.to_string()) + .or_insert(HashSet::new()) + .insert(session_id.clone()); + self.items.push_back(( + consumer_id.to_string(), + p::OutgoingMessage::SessionStarted { + peer_id: producer_id.to_string(), + session_id: session_id.clone(), + }, + )); + self.items.push_back(( + producer_id.to_string(), + p::OutgoingMessage::StartSession { + peer_id: consumer_id.to_string(), + session_id: session_id.clone(), + }, + )); + + info!(id = %session_id, producer_id = %producer_id, consumer_id = %consumer_id, "started a session"); + + Ok(()) + } +} + +impl Stream for Handler { + type Item = (String, p::OutgoingMessage); + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> { + loop { + let this = self.as_mut().project(); + + if let Some(item) = this.items.pop_front() { + break Poll::Ready(Some(item)); + } + + match ready!(this.stream.poll_next(cx)) { + Some((peer_id, msg)) => { + if let Some(msg) = msg { + if let Err(err) = self.as_mut().handle(&peer_id, msg) { + self.items.push_back(( + peer_id.to_string(), + p::OutgoingMessage::Error { + details: err.to_string(), + }, + )); + } + } else { + self.remove_peer(&peer_id); + } + } + None => { + break Poll::Ready(None); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::channel::mpsc; + use serde_json::json; + + async fn new_peer( + tx: &mut mpsc::UnboundedSender<(String, Option<p::IncomingMessage>)>, + handler: &mut Handler, + peer_id: &str, + ) { + tx.send((peer_id.to_string(), Some(p::IncomingMessage::NewPeer))) + .await + .unwrap(); + + let res = handler.next().await.unwrap(); + assert_eq!( + res, + ( + peer_id.to_string(), + p::OutgoingMessage::Welcome { + peer_id: peer_id.to_string() + } + ) + ); + } + + #[async_std::test] + async fn test_register_producer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + tx.send(( + "producer".to_string(), + Some(p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + })), + )) + .await + .unwrap(); + } + + #[async_std::test] + async fn test_list_producers() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + meta: Some(json!({"display-name":"foobar".to_string()})), + roles: vec![p::PeerRole::Producer], + peer_id: None, + }); + + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + let message = p::IncomingMessage::List; + tx.send(("listener".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "listener"); + assert_eq!( + sent_message, + p::OutgoingMessage::List { + producers: vec![p::Peer { + id: "producer".to_string(), + meta: Some(json!( + {"display-name": "foobar".to_string() + })), + }] + } + ); + } + + #[async_std::test] + async fn test_welcome() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "consumer").await; + } + + #[async_std::test] + async fn test_listener() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + new_peer(&mut tx, &mut handler, "listener").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Listener], + meta: None, + peer_id: None, + }); + tx.send(("listener".to_string(), Some(message))) + .await + .unwrap(); + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: Some(json!({ + "display-name": "foobar".to_string(), + })), + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "listener"); + assert_eq!( + sent_message, + p::OutgoingMessage::PeerStatusChanged(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + peer_id: Some("producer".to_string()), + meta: Some(json!({ + "display-name": Some("foobar".to_string()), + } + )) + }) + ); + } + + #[async_std::test] + async fn test_start_session() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing {:?}", sent_message), + }; + + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::StartSession { + peer_id: "consumer".to_string(), + session_id: session_id.to_string(), + } + ); + } + + #[async_std::test] + async fn test_remove_peer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + assert_eq!( + handler.next().await.unwrap(), + ( + "producer".into(), + p::OutgoingMessage::StartSession { + peer_id: "consumer".into(), + session_id: session_id.clone() + } + ) + ); + + new_peer(&mut tx, &mut handler, "listener").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Listener], + meta: None, + peer_id: None, + }); + tx.send(("listener".to_string(), Some(message))) + .await + .unwrap(); + let _ = handler.next().await.unwrap(); + + handler.remove_peer("producer"); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { session_id }) + ); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "listener"); + assert_eq!( + sent_message, + p::OutgoingMessage::PeerStatusChanged(PeerStatus { + roles: vec![], + peer_id: Some("producer".to_string()), + meta: Default::default() + }) + ); + } + + #[async_std::test] + async fn test_end_session_consumer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }); + + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id + }) + ); + } + + #[async_std::test] + async fn test_disconnect_consumer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + tx.send(("consumer".to_string(), None)).await.unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id + }) + ); + } + + #[async_std::test] + async fn test_end_session_producer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { session_id }) + ); + } + + #[async_std::test] + async fn test_end_session_twice() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + // The consumer ends the session + let message = p::IncomingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone() + }) + ); + + let message = p::IncomingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Error { + details: format!("Session {session_id} doesn't exist") + } + ); + } + + #[async_std::test] + async fn test_sdp_exchange() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Sdp(p::SdpMessage::Offer { + sdp: "offer".to_string(), + }), + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Sdp(p::SdpMessage::Offer { + sdp: "offer".to_string() + }) + }) + ); + } + + #[async_std::test] + async fn test_ice_exchange() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Ice { + candidate: "candidate".to_string(), + sdp_m_line_index: 42, + }, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Ice { + candidate: "candidate".to_string(), + sdp_m_line_index: 42 + } + }) + ); + + let message = p::IncomingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Ice { + candidate: "candidate".to_string(), + sdp_m_line_index: 42, + }, + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Peer(p::PeerMessage { + session_id: session_id.clone(), + peer_message: p::PeerMessageInner::Ice { + candidate: "candidate".to_string(), + sdp_m_line_index: 42 + } + }) + ); + } + + #[async_std::test] + async fn test_sdp_exchange_wrong_direction_offer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::Peer(p::PeerMessage { + session_id, + peer_message: p::PeerMessageInner::Sdp(p::SdpMessage::Offer { + sdp: "offer".to_string(), + }), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let response = handler.next().await.unwrap(); + + assert_eq!(response, + ( + "consumer".into(), + p::OutgoingMessage::Error { + details: r#"cannot forward offer from "consumer" to "producer" as "consumer" is not the producer"#.into() + } + ) + ); + } + + #[async_std::test] + async fn test_start_session_no_producer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Error { + details: "Peer 'producer' hasn't been welcomed".into() + } + ); + } + + #[async_std::test] + async fn test_stop_producing() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::StartSession { + peer_id: "consumer".to_string(), + session_id: session_id.clone(), + } + ); + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }) + ); + } + + #[async_std::test] + async fn test_unregistering_with_listeners() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "listener").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Listener], + meta: None, + peer_id: None, + }); + tx.send(("listener".to_string(), Some(message))) + .await + .unwrap(); + let _ = handler.next().await.unwrap(); + + new_peer(&mut tx, &mut handler, "producer").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "listener"); + assert_eq!( + sent_message, + p::OutgoingMessage::PeerStatusChanged(PeerStatus { + roles: vec![p::PeerRole::Producer], + peer_id: Some("producer".to_string()), + meta: Default::default() + }) + ); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing {:?}", sent_message), + }; + + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "producer"); + assert_eq!( + sent_message, + p::OutgoingMessage::StartSession { + peer_id: "consumer".to_string(), + session_id: session_id.clone(), + } + ); + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + assert_eq!( + handler.next().await.unwrap(), + ( + "consumer".into(), + p::OutgoingMessage::EndSession(p::EndSessionMessage { + session_id: session_id.clone(), + }) + ) + ); + + assert_eq!( + handler.next().await.unwrap(), + ( + "listener".into(), + p::OutgoingMessage::PeerStatusChanged(PeerStatus { + roles: vec![], + peer_id: Some("producer".to_string()), + meta: Default::default() + }) + ) + ); + } + + #[async_std::test] + async fn test_start_session_no_consumer() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + + assert_eq!(peer_id, "consumer"); + assert_eq!( + sent_message, + p::OutgoingMessage::Error { + details: "Peer 'consumer' hasn't been welcomed".into() + } + ); + } + + #[async_std::test] + async fn test_start_session_twice() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + meta: Some(json!( {"display-name": "foobar".to_string() })), + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + + new_peer(&mut tx, &mut handler, "consumer").await; + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session0_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let _ = handler.next().await.unwrap(); + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + + tx.send(("consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "consumer"); + let session1_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + assert_ne!(session0_id, session1_id); + } + + #[async_std::test] + async fn test_start_session_stop_producing() { + let (mut tx, rx) = mpsc::unbounded(); + let mut handler = Handler::new(Box::pin(rx)); + + new_peer(&mut tx, &mut handler, "producer").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer, p::PeerRole::Listener], + meta: None, + peer_id: None, + }); + tx.send(("producer".to_string(), Some(message))) + .await + .unwrap(); + let _ = handler.next().await.unwrap(); + + new_peer(&mut tx, &mut handler, "producer-consumer").await; + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Producer], + ..Default::default() + }); + tx.send(("producer-consumer".to_string(), Some(message))) + .await + .unwrap(); + handler.next().await.unwrap(); + + let message = p::IncomingMessage::StartSession(p::StartSessionMessage { + peer_id: "producer".to_string(), + }); + tx.send(("producer-consumer".to_string(), Some(message))) + .await + .unwrap(); + let (peer_id, sent_message) = handler.next().await.unwrap(); + assert_eq!(peer_id, "producer-consumer"); + let session0_id = match sent_message { + p::OutgoingMessage::SessionStarted { + ref peer_id, + ref session_id, + } => { + assert_eq!(peer_id, "producer"); + session_id.to_string() + } + _ => panic!("SessionStarted message missing"), + }; + + let message = p::IncomingMessage::SetPeerStatus(p::PeerStatus { + roles: vec![p::PeerRole::Listener], + ..Default::default() + }); + tx.send(("producer-consumer".to_string(), Some(message))) + .await + .unwrap(); + handler.next().await.unwrap(); + + let message = p::IncomingMessage::List; + tx.send(("producer-consumer".to_string(), Some(message))) + .await + .unwrap(); + + handler.next().await.unwrap(); + handler + .sessions + .get(&session0_id) + .expect("Session should remain"); + } +} diff --git a/net/webrtc/signalling/src/lib.rs b/net/webrtc/signalling/src/lib.rs new file mode 100644 index 00000000..068798b4 --- /dev/null +++ b/net/webrtc/signalling/src/lib.rs @@ -0,0 +1,2 @@ +pub mod handlers; +pub mod server; diff --git a/net/webrtc/signalling/src/server/mod.rs b/net/webrtc/signalling/src/server/mod.rs new file mode 100644 index 00000000..8ece1efd --- /dev/null +++ b/net/webrtc/signalling/src/server/mod.rs @@ -0,0 +1,218 @@ +use anyhow::Error; +use async_std::task; +use async_tungstenite::tungstenite::Message as WsMessage; +use futures::channel::mpsc; +use futures::prelude::*; +use futures::{AsyncRead, AsyncWrite}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use tracing::{info, instrument, trace, warn}; + +struct Peer { + receive_task_handle: task::JoinHandle<()>, + send_task_handle: task::JoinHandle<Result<(), Error>>, + sender: mpsc::Sender<String>, +} + +struct State { + tx: Option<mpsc::Sender<(String, Option<String>)>>, + peers: HashMap<String, Peer>, +} + +#[derive(Clone)] +pub struct Server { + state: Arc<Mutex<State>>, +} + +#[derive(thiserror::Error, Debug)] +pub enum ServerError { + #[error("error during handshake {0}")] + Handshake(#[from] async_tungstenite::tungstenite::Error), +} + +impl Server { + #[instrument(level = "debug", skip(factory))] + pub fn spawn< + I: for<'a> Deserialize<'a>, + O: Serialize + std::fmt::Debug, + Factory: FnOnce(Pin<Box<dyn Stream<Item = (String, Option<I>)> + Send>>) -> St, + St: Stream<Item = (String, O)>, + >( + factory: Factory, + ) -> Self + where + O: Serialize + std::fmt::Debug, + St: Send + Unpin + 'static, + { + let (tx, rx) = mpsc::channel::<(String, Option<String>)>(1000); + let mut handler = factory(Box::pin(rx.filter_map(|(peer_id, msg)| async move { + if let Some(msg) = msg { + match serde_json::from_str::<I>(&msg) { + Ok(msg) => Some((peer_id, Some(msg))), + Err(err) => { + warn!("Failed to parse incoming message: {} ({})", err, msg); + None + } + } + } else { + Some((peer_id, None)) + } + }))); + + let state = Arc::new(Mutex::new(State { + tx: Some(tx), + peers: HashMap::new(), + })); + + let state_clone = state.clone(); + let _ = task::spawn(async move { + while let Some((peer_id, msg)) = handler.next().await { + match serde_json::to_string(&msg) { + Ok(msg) => { + if let Some(peer) = state_clone.lock().unwrap().peers.get_mut(&peer_id) { + let mut sender = peer.sender.clone(); + task::spawn(async move { + let _ = sender.send(msg).await; + }); + } + } + Err(err) => { + warn!("Failed to serialize outgoing message: {}", err); + } + } + } + }); + + Self { state } + } + + #[instrument(level = "debug", skip(state))] + fn remove_peer(state: Arc<Mutex<State>>, peer_id: &str) { + if let Some(mut peer) = state.lock().unwrap().peers.remove(peer_id) { + let peer_id = peer_id.to_string(); + task::spawn(async move { + peer.sender.close_channel(); + if let Err(err) = peer.send_task_handle.await { + trace!(peer_id = %peer_id, "Error while joining send task: {}", err); + } + peer.receive_task_handle.await; + }); + } + } + + #[instrument(level = "debug", skip(self, stream))] + pub async fn accept_async<S: 'static>(&mut self, stream: S) -> Result<String, ServerError> + where + S: AsyncRead + AsyncWrite + Unpin + Send, + { + let ws = match async_tungstenite::accept_async(stream).await { + Ok(ws) => ws, + Err(err) => { + warn!("Error during the websocket handshake: {}", err); + return Err(ServerError::Handshake(err)); + } + }; + + let this_id = uuid::Uuid::new_v4().to_string(); + info!(this_id = %this_id, "New WebSocket connection"); + + // 1000 is completely arbitrary, we simply don't want infinite piling + // up of messages as with unbounded + let (websocket_sender, mut websocket_receiver) = mpsc::channel::<String>(1000); + + let this_id_clone = this_id.clone(); + let (mut ws_sink, mut ws_stream) = ws.split(); + let send_task_handle = task::spawn(async move { + loop { + match async_std::future::timeout( + std::time::Duration::from_secs(30), + websocket_receiver.next(), + ) + .await + { + Ok(Some(msg)) => { + trace!(this_id = %this_id_clone, "sending {}", msg); + ws_sink.send(WsMessage::Text(msg)).await?; + } + Ok(None) => { + break; + } + Err(_) => { + trace!(this_id = %this_id_clone, "timeout, sending ping"); + ws_sink.send(WsMessage::Ping(vec![])).await?; + } + } + } + + ws_sink.send(WsMessage::Close(None)).await?; + ws_sink.close().await?; + + Ok::<(), Error>(()) + }); + + let mut tx = self.state.lock().unwrap().tx.clone(); + let this_id_clone = this_id.clone(); + let state_clone = self.state.clone(); + let receive_task_handle = task::spawn(async move { + if let Some(tx) = tx.as_mut() { + if let Err(err) = tx + .send(( + this_id_clone.clone(), + Some( + serde_json::json!({ + "type": "newPeer", + }) + .to_string(), + ), + )) + .await + { + warn!(this = %this_id_clone, "Error handling message: {:?}", err); + } + } + while let Some(msg) = ws_stream.next().await { + info!("Received message {msg:?}"); + match msg { + Ok(WsMessage::Text(msg)) => { + if let Some(tx) = tx.as_mut() { + if let Err(err) = tx.send((this_id_clone.clone(), Some(msg))).await { + warn!(this = %this_id_clone, "Error handling message: {:?}", err); + } + } + } + Ok(WsMessage::Close(reason)) => { + info!(this_id = %this_id_clone, "connection closed: {:?}", reason); + break; + } + Ok(WsMessage::Pong(_)) => { + continue; + } + Ok(_) => warn!(this_id = %this_id_clone, "Unsupported message type"), + Err(err) => { + warn!(this_id = %this_id_clone, "recv error: {}", err); + break; + } + } + } + + if let Some(tx) = tx.as_mut() { + let _ = tx.send((this_id_clone.clone(), None)).await; + } + + Self::remove_peer(state_clone, &this_id_clone); + }); + + self.state.lock().unwrap().peers.insert( + this_id.clone(), + Peer { + receive_task_handle, + send_task_handle, + sender: websocket_sender, + }, + ); + + Ok(this_id) + } +} diff --git a/net/webrtc/www/index.html b/net/webrtc/www/index.html new file mode 100644 index 00000000..e533c4ff --- /dev/null +++ b/net/webrtc/www/index.html @@ -0,0 +1,38 @@ +<!DOCTYPE html> +<!-- + vim: set sts=2 sw=2 et : + + + Demo Javascript app for negotiating and streaming a sendrecv webrtc stream + with a GStreamer app. Runs only in passive mode, i.e., responds to offers + with answers, exchanges ICE candidates, and streams. + + Author: Nirbheek Chauhan <nirbheek@centricular.com> +--> +<html> + <head> + <style> + .error { color: red; } + </style> + <link rel="stylesheet" type="text/css" href="theme.css"> + <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script> + <script src="keyboard.js"></script> + <script src="input.js"></script> + <script src="webrtc.js"></script> + <script>window.onload = setup;</script> + </head> + + <body> + <div class="holygrail-body"> + <div class="content"> + <div id="sessions"> + </div> + <div id="image-holder"> + <img id="image"></img> + </div> + </div> + <ul class="nav" id="camera-list"> + </ul> + </div> + </body> +</html> diff --git a/net/webrtc/www/input.js b/net/webrtc/www/input.js new file mode 100644 index 00000000..805f3557 --- /dev/null +++ b/net/webrtc/www/input.js @@ -0,0 +1,482 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*global GamepadManager*/ +/*eslint no-unused-vars: ["error", { "vars": "local" }]*/ + + +class Input { + /** + * Input handling for WebRTC web app + * + * @constructor + * @param {Element} [element] + * Video element to attach events to + * @param {function} [send] + * Function used to send input events to server. + */ + constructor(element, send) { + /** + * @type {Element} + */ + this.element = element; + + /** + * @type {function} + */ + this.send = send; + + /** + * @type {boolean} + */ + this.mouseRelative = false; + + /** + * @type {Object} + */ + this.m = null; + + /** + * @type {Keyboard} + */ + this.keyboard = null; + + /** + * @type {GamepadManager} + */ + this.gamepadManager = null; + + /** + * @type {Integer} + */ + this.x = 0; + + /** + * @type {Integer} + */ + this.y = 0; + + /** + * @type {Integer} + */ + this.lastTouch = 0; + + /** + * @type {function} + */ + this.ongamepadconnected = null; + + /** + * @type {function} + */ + this.ongamepaddisconneceted = null; + + /** + * List of attached listeners, record keeping used to detach all. + * @type {Array} + */ + this.listeners = []; + + /** + * @type {function} + */ + this.onresizeend = null; + + // internal variables used by resize start/end functions. + this._rtime = null; + this._rtimeout = false; + this._rdelta = 200; + } + + /** + * Handles mouse button and motion events and sends them to WebRTC app. + * @param {MouseEvent} event + */ + _mouseButtonMovement(event) { + const down = (event.type === 'mousedown' ? 1 : 0); + var data = {}; + + if (event.type === 'mousemove' && !this.m) return; + + if (!document.pointerLockElement) { + if (this.mouseRelative) + event.target.requestPointerLock(); + } + + // Hotkey to enable pointer lock, CTRL-SHIFT-LeftButton + if (down && event.button === 0 && event.ctrlKey && event.shiftKey) { + event.target.requestPointerLock(); + return; + } + + if (document.pointerLockElement) { + // FIXME - mark as relative! + console.warn("FIXME: Make event relative!") + this.x = event.movementX; + this.y = event.movementY; + } else if (event.type === 'mousemove') { + this.x = this._clientToServerX(event.clientX); + this.y = this._clientToServerY(event.clientY); + data["event"] = "MouseMove" + } + + if (event.type === 'mousedown') { + data["event"] = "MouseButtonPress"; + } else if (event.type === 'mouseup') { + data["event"] = "MouseButtonRelease"; + } + + if (event.type === 'mousedown' || event.type === 'mouseup') { + data["button"] = event.button + 1; + } + + data["x"] = this.x; + data["y"] = this.y; + data["modifier_state"] = this._modifierState(event); + + this.send(data); + } + + /** + * Handles touch events and sends them to WebRTC app. + * @param {TouchEvent} event + */ + _touch(event) { + var mod_state = this._modifierState(event); + + // Use TouchUp for cancelled touch points + if (event.type === 'touchcancel') { + let data = {}; + + data["event"] = "TouchUp"; + data["identifier"] = event.changedTouches[0].identifier; + data["x"] = this._clientToServerX(event.changedTouches[0].clientX); + data["y"] = this._clientToServerY(event.changedTouches[0].clientY); + data["modifier_state"] = mod_state; + + this.send(data); + return; + } + + if (event.type === 'touchstart') { + var event_name = "TouchDown"; + } else if (event.type === 'touchmove') { + var event_name = "TouchMotion"; + } else if (event.type === 'touchend') { + var event_name = "TouchUp"; + } + + for (let touch of event.changedTouches) { + let data = {}; + + data["event"] = event_name; + data["identifier"] = touch.identifier; + data["x"] = this._clientToServerX(touch.clientX); + data["y"] = this._clientToServerY(touch.clientY); + data["modifier_state"] = mod_state; + + if (event.type !== 'touchend') { + if ('force' in touch) { + data["pressure"] = touch.force; + } else { + data["pressure"] = NaN; + } + } + + this.send(data); + } + + if (event.timeStamp > this.lastTouch) { + let data = {}; + + data["event"] = "TouchFrame"; + data["modifier_state"] = mod_state; + + this.send(data); + this.lastTouch = event.timeStamp; + } + + event.preventDefault(); + } + + /** + * Handles mouse wheel events and sends them to WebRTC app. + * @param {MouseEvent} event + */ + _wheel(event) { + let data = { + "event": "MouseScroll", + "x": this.x, + "y": this.y, + "delta_x": -event.deltaX, + "delta_y": -event.deltaY, + "modifier_state": this._modifierState(event), + }; + + this.send(data); + + event.preventDefault(); + } + + /** + * Captures mouse context menu (right-click) event and prevents event propagation. + * @param {MouseEvent} event + */ + _contextMenu(event) { + event.preventDefault(); + } + + /** + * Sends WebRTC app command to hide the remote pointer when exiting pointer lock. + */ + _exitPointerLock() { + document.exitPointerLock(); + } + + /** + * constructs the string representation for the active modifiers on the event + */ + _modifierState(event) { + let masks = [] + if (event.altKey) masks.push("alt-mask"); + if (event.ctrlKey) masks.push("control-mask"); + if (event.metaKey) masks.push("meta-mask"); + if (event.shiftKey) masks.push("shift-mask"); + return masks.join('+') + } + + /** + * Captures display and video dimensions required for computing mouse pointer position. + * This should be fired whenever the window size changes. + */ + _windowMath() { + const windowW = this.element.offsetWidth; + const windowH = this.element.offsetHeight; + const frameW = this.element.videoWidth; + const frameH = this.element.videoHeight; + + const multi = Math.min(windowW / frameW, windowH / frameH); + const vpWidth = frameW * multi; + const vpHeight = (frameH * multi); + + var elem = this.element; + var offsetLeft = 0; + var offsetTop = 0; + do { + if (!isNaN(elem.offsetLeft)) { + offsetLeft += elem.offsetLeft; + } + + if (!isNaN(elem.offsetTop)) { + offsetTop += elem.offsetTop; + } + } while (elem = elem.offsetParent); + + this.m = { + mouseMultiX: frameW / vpWidth, + mouseMultiY: frameH / vpHeight, + mouseOffsetX: Math.max((windowW - vpWidth) / 2.0, 0), + mouseOffsetY: Math.max((windowH - vpHeight) / 2.0, 0), + offsetLeft: offsetLeft, + offsetTop: offsetTop, + scrollX: window.scrollX, + scrollY: window.scrollY, + frameW, + frameH, + }; + } + + /** + * Translates pointer position X based on current window math. + * @param {Integer} clientX + */ + _clientToServerX(clientX) { + var serverX = Math.round((clientX - this.m.mouseOffsetX - this.m.offsetLeft + this.m.scrollX) * this.m.mouseMultiX); + + if (serverX === this.m.frameW - 1) serverX = this.m.frameW; + if (serverX > this.m.frameW) serverX = this.m.frameW; + if (serverX < 0) serverX = 0; + + return serverX; + } + + /** + * Translates pointer position Y based on current window math. + * @param {Integer} clientY + */ + _clientToServerY(clientY) { + let serverY = Math.round((clientY - this.m.mouseOffsetY - this.m.offsetTop + this.m.scrollY) * this.m.mouseMultiY); + + if (serverY === this.m.frameH - 1) serverY = this.m.frameH; + if (serverY > this.m.frameH) serverY = this.m.frameH; + if (serverY < 0) serverY = 0; + + return serverY; + } + + /** + * When fullscreen is entered, request keyboard and pointer lock. + */ + _onFullscreenChange() { + if (document.fullscreenElement !== null) { + // Enter fullscreen + this.requestKeyboardLock(); + this.element.requestPointerLock(); + } + // Reset local keyboard. When holding to exit full-screen the escape key can get stuck. + this.keyboard.reset(); + + // Reset stuck keys on server side. + // FIXME: How to implement resetting keyboard with the GstNavigation interface + // this.send("kr"); + } + + /** + * Called when window is being resized, used to detect when resize ends so new resolution can be sent. + */ + _resizeStart() { + this._rtime = new Date(); + if (this._rtimeout === false) { + this._rtimeout = true; + setTimeout(() => { this._resizeEnd() }, this._rdelta); + } + } + + /** + * Called in setTimeout loop to detect if window is done being resized. + */ + _resizeEnd() { + if (new Date() - this._rtime < this._rdelta) { + setTimeout(() => { this._resizeEnd() }, this._rdelta); + } else { + this._rtimeout = false; + if (this.onresizeend !== null) { + this.onresizeend(); + } + } + } + + /** + * Attaches input event handles to docuemnt, window and element. + */ + attach() { + this.listeners.push(addListener(this.element, 'resize', this._windowMath, this)); + this.listeners.push(addListener(this.element, 'wheel', this._wheel, this)); + this.listeners.push(addListener(this.element, 'contextmenu', this._contextMenu, this)); + this.listeners.push(addListener(this.element.parentElement, 'fullscreenchange', this._onFullscreenChange, this)); + this.listeners.push(addListener(window, 'resize', this._windowMath, this)); + this.listeners.push(addListener(window, 'resize', this._resizeStart, this)); + + if ('ontouchstart' in window) { + console.warning("FIXME: Enabling mouse pointer display for touch devices."); + } else { + this.listeners.push(addListener(this.element, 'mousemove', this._mouseButtonMovement, this)); + this.listeners.push(addListener(this.element, 'mousedown', this._mouseButtonMovement, this)); + this.listeners.push(addListener(this.element, 'mouseup', this._mouseButtonMovement, this)); + } + + this.listeners.push(addListener(this.element, 'touchstart', this._touch, this)); + this.listeners.push(addListener(this.element, 'touchend', this._touch, this)); + this.listeners.push(addListener(this.element, 'touchmove', this._touch, this)); + this.listeners.push(addListener(this.element, 'touchcancel', this._touch, this)); + + // Adjust for scroll offset + this.listeners.push(addListener(window, 'scroll', () => { + this.m.scrollX = window.scrollX; + this.m.scrollY = window.scrollY; + }, this)); + + // Using guacamole keyboard because it has the keysym translations. + this.keyboard = new Keyboard(this.element); + this.keyboard.onkeydown = (keysym, state) => { + this.send({"event": "KeyPress", "key": keysym, "modifier_state": state}); + }; + this.keyboard.onkeyup = (keysym, state) => { + this.send({"event": "KeyRelease", "key": keysym, "modifier_state": state}); + }; + + this._windowMath(); + } + + detach() { + removeListeners(this.listeners); + this._exitPointerLock(); + if (this.keyboard) { + this.keyboard.onkeydown = null; + this.keyboard.onkeyup = null; + this.keyboard.reset(); + delete this.keyboard; + // FIXME: How to implement resetting keyboard with the GstNavigation interface + // this.send("kr"); + } + } + + /** + * Request keyboard lock, must be in fullscreen mode to work. + */ + requestKeyboardLock() { + // event codes: https://www.w3.org/TR/uievents-code/#key-alphanumeric-writing-system + const keys = [ + "AltLeft", + "AltRight", + "Tab", + "Escape", + "ContextMenu", + "MetaLeft", + "MetaRight" + ]; + console.log("requesting keyboard lock"); + navigator.keyboard.lock(keys).then( + () => { + console.log("keyboard lock success"); + } + ).catch( + (e) => { + console.log("keyboard lock failed: ", e); + } + ) + } + + getWindowResolution() { + return [ + parseInt(this.element.offsetWidth * window.devicePixelRatio), + parseInt(this.element.offsetHeight * window.devicePixelRatio) + ]; + } +} + +/** + * Helper function to keep track of attached event listeners. + * @param {Object} obj + * @param {string} name + * @param {function} func + * @param {Object} ctx + */ +function addListener(obj, name, func, ctx) { + const newFunc = ctx ? func.bind(ctx) : func; + obj.addEventListener(name, newFunc); + + return [obj, name, newFunc]; +} + +/** + * Helper function to remove all attached event listeners. + * @param {Array} listeners + */ +function removeListeners(listeners) { + for (const listener of listeners) + listener[0].removeEventListener(listener[1], listener[2]); +} diff --git a/net/webrtc/www/keyboard.js b/net/webrtc/www/keyboard.js new file mode 100644 index 00000000..4305cd1a --- /dev/null +++ b/net/webrtc/www/keyboard.js @@ -0,0 +1,3302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Provides cross-browser and cross-keyboard keyboard for a specific element. + * Browser and keyboard layout variation is abstracted away, providing events + * which represent keys as their corresponding X11 keysym. + * + * @constructor + * @param {Element} element The Element to use to provide keyboard events. + */ +Keyboard = function(element) { + + /** + * Reference to this Keyboard. + * @private + */ + var guac_keyboard = this; + + /** + * Fired whenever the user presses a key with the element associated + * with this Keyboard in focus. + * + * @event + * @param {Number} keysym The keysym of the key being pressed. + * @param {String} modifier_state The string representation of + * all active modifiers. + * @return {Boolean} true if the key event should be allowed through to the + * browser, false otherwise. + */ + this.onkeydown = null; + + /** + * Fired whenever the user releases a key with the element associated + * with this Keyboard in focus. + * + * @event + * @param {Number} keysym The keysym of the key being released. + * @param {String} modifier_state The string representation of + * all active modifiers. + */ + this.onkeyup = null; + + /** + * A key event having a corresponding timestamp. This event is non-specific. + * Its subclasses should be used instead when recording specific key + * events. + * + * @private + * @constructor + */ + var KeyEvent = function() { + + /** + * Reference to this key event. + */ + var key_event = this; + + /** + * An arbitrary timestamp in milliseconds, indicating this event's + * position in time relative to other events. + * + * @type {Number} + */ + this.timestamp = new Date().getTime(); + + /** + * Whether the default action of this key event should be prevented. + * + * @type {Boolean} + */ + this.defaultPrevented = false; + + /** + * The keysym of the key associated with this key event, as determined + * by a best-effort guess using available event properties and keyboard + * state. + * + * @type {Number} + */ + this.keysym = null; + + /** + * Whether the keysym value of this key event is known to be reliable. + * If false, the keysym may still be valid, but it's only a best guess, + * and future key events may be a better source of information. + * + * @type {Boolean} + */ + this.reliable = false; + + /** + * Returns the number of milliseconds elapsed since this event was + * received. + * + * @return {Number} The number of milliseconds elapsed since this + * event was received. + */ + this.getAge = function() { + return new Date().getTime() - key_event.timestamp; + }; + + }; + + /** + * Information related to the pressing of a key, which need not be a key + * associated with a printable character. The presence or absence of any + * information within this object is browser-dependent. + * + * @private + * @constructor + * @augments Keyboard.KeyEvent + * @param {Number} keyCode The JavaScript key code of the key pressed. + * @param {String} keyIdentifier The legacy DOM3 "keyIdentifier" of the key + * pressed, as defined at: + * http://www.w3.org/TR/2009/WD-DOM-Level-3-Events-20090908/#events-Events-KeyboardEvent + * @param {String} key The standard name of the key pressed, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * @param {Number} location The location on the keyboard corresponding to + * the key pressed, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + */ + var KeydownEvent = function(keyCode, keyIdentifier, key, location) { + + // We extend KeyEvent + KeyEvent.apply(this); + + /** + * The JavaScript key code of the key pressed. + * + * @type {Number} + */ + this.keyCode = keyCode; + + /** + * The legacy DOM3 "keyIdentifier" of the key pressed, as defined at: + * http://www.w3.org/TR/2009/WD-DOM-Level-3-Events-20090908/#events-Events-KeyboardEvent + * + * @type {String} + */ + this.keyIdentifier = keyIdentifier; + + /** + * The standard name of the key pressed, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * + * @type {String} + */ + this.key = key; + + /** + * The location on the keyboard corresponding to the key pressed, as + * defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * + * @type {Number} + */ + this.location = location; + + // If key is known from keyCode or DOM3 alone, use that + this.keysym = keysym_from_key_identifier(key, location) + || keysym_from_keycode(keyCode, location); + + // DOM3 and keyCode are reliable sources if the corresponding key is + // not a printable key + if (this.keysym && !isPrintable(this.keysym)) + this.reliable = true; + + // Use legacy keyIdentifier as a last resort, if it looks sane + if (!this.keysym && key_identifier_sane(keyCode, keyIdentifier)) + this.keysym = keysym_from_key_identifier(keyIdentifier, location, guac_keyboard.modifiers.shift); + + // Determine whether default action for Alt+combinations must be prevented + var prevent_alt = !guac_keyboard.modifiers.ctrl + && !(navigator && navigator.platform && navigator.platform.match(/^mac/i)); + + // Determine whether default action for Ctrl+combinations must be prevented + var prevent_ctrl = !guac_keyboard.modifiers.alt; + + // We must rely on the (potentially buggy) keyIdentifier if preventing + // the default action is important + if ((prevent_ctrl && guac_keyboard.modifiers.ctrl) + || (prevent_alt && guac_keyboard.modifiers.alt) + || guac_keyboard.modifiers.meta + || guac_keyboard.modifiers.hyper) + this.reliable = true; + + // Record most recently known keysym by associated key code + recentKeysym[keyCode] = this.keysym; + + }; + + KeydownEvent.prototype = new KeyEvent(); + + /** + * Information related to the pressing of a key, which MUST be + * associated with a printable character. The presence or absence of any + * information within this object is browser-dependent. + * + * @private + * @constructor + * @augments Keyboard.KeyEvent + * @param {Number} charCode The Unicode codepoint of the character that + * would be typed by the key pressed. + */ + var KeypressEvent = function(charCode) { + + // We extend KeyEvent + KeyEvent.apply(this); + + /** + * The Unicode codepoint of the character that would be typed by the + * key pressed. + * + * @type {Number} + */ + this.charCode = charCode; + + // Pull keysym from char code + this.keysym = keysym_from_charcode(charCode); + + // Keypress is always reliable + this.reliable = true; + + }; + + KeypressEvent.prototype = new KeyEvent(); + + /** + * Information related to the pressing of a key, which need not be a key + * associated with a printable character. The presence or absence of any + * information within this object is browser-dependent. + * + * @private + * @constructor + * @augments Keyboard.KeyEvent + * @param {Number} keyCode The JavaScript key code of the key released. + * @param {String} keyIdentifier The legacy DOM3 "keyIdentifier" of the key + * released, as defined at: + * http://www.w3.org/TR/2009/WD-DOM-Level-3-Events-20090908/#events-Events-KeyboardEvent + * @param {String} key The standard name of the key released, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * @param {Number} location The location on the keyboard corresponding to + * the key released, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + */ + var KeyupEvent = function(keyCode, keyIdentifier, key, location) { + + // We extend KeyEvent + KeyEvent.apply(this); + + /** + * The JavaScript key code of the key released. + * + * @type {Number} + */ + this.keyCode = keyCode; + + /** + * The legacy DOM3 "keyIdentifier" of the key released, as defined at: + * http://www.w3.org/TR/2009/WD-DOM-Level-3-Events-20090908/#events-Events-KeyboardEvent + * + * @type {String} + */ + this.keyIdentifier = keyIdentifier; + + /** + * The standard name of the key released, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * + * @type {String} + */ + this.key = key; + + /** + * The location on the keyboard corresponding to the key released, as + * defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + * + * @type {Number} + */ + this.location = location; + + // If key is known from keyCode or DOM3 alone, use that + this.keysym = recentKeysym[keyCode] + || keysym_from_keycode(keyCode, location) + || keysym_from_key_identifier(key, location); // keyCode is still more reliable for keyup when dead keys are in use + + // Keyup is as reliable as it will ever be + this.reliable = true; + + }; + + KeyupEvent.prototype = new KeyEvent(); + + /** + * An array of recorded events, which can be instances of the private + * KeydownEvent, KeypressEvent, and KeyupEvent classes. + * + * @private + * @type {KeyEvent[]} + */ + var eventLog = []; + + /** + * Map of known JavaScript keycodes which do not map to typable characters + * to their X11 keysym equivalents. + * @private + */ + var keycodeKeysyms = { + 8: [0xFF08], // backspace + 9: [0xFF09], // tab + 12: [0xFF0B, 0xFF0B, 0xFF0B, 0xFFB5], // clear / KP 5 + 13: [0xFF0D], // enter + 16: [0xFFE1, 0xFFE1, 0xFFE2], // shift + 17: [0xFFE3, 0xFFE3, 0xFFE4], // ctrl + 18: [0xFFE9, 0xFFE9, 0xFE03], // alt + 19: [0xFF13], // pause/break + 20: [0xFFE5], // caps lock + 27: [0xFF1B], // escape + 32: [0x0020], // space + 33: [0xFF55, 0xFF55, 0xFF55, 0xFFB9], // page up / KP 9 + 34: [0xFF56, 0xFF56, 0xFF56, 0xFFB3], // page down / KP 3 + 35: [0xFF57, 0xFF57, 0xFF57, 0xFFB1], // end / KP 1 + 36: [0xFF50, 0xFF50, 0xFF50, 0xFFB7], // home / KP 7 + 37: [0xFF51, 0xFF51, 0xFF51, 0xFFB4], // left arrow / KP 4 + 38: [0xFF52, 0xFF52, 0xFF52, 0xFFB8], // up arrow / KP 8 + 39: [0xFF53, 0xFF53, 0xFF53, 0xFFB6], // right arrow / KP 6 + 40: [0xFF54, 0xFF54, 0xFF54, 0xFFB2], // down arrow / KP 2 + 45: [0xFF63, 0xFF63, 0xFF63, 0xFFB0], // insert / KP 0 + 46: [0xFFFF, 0xFFFF, 0xFFFF, 0xFFAE], // delete / KP decimal + 91: [0xFFEB], // left window key (hyper_l) + 92: [0xFF67], // right window key (menu key?) + 93: null, // select key + 96: [0xFFB0], // KP 0 + 97: [0xFFB1], // KP 1 + 98: [0xFFB2], // KP 2 + 99: [0xFFB3], // KP 3 + 100: [0xFFB4], // KP 4 + 101: [0xFFB5], // KP 5 + 102: [0xFFB6], // KP 6 + 103: [0xFFB7], // KP 7 + 104: [0xFFB8], // KP 8 + 105: [0xFFB9], // KP 9 + 106: [0xFFAA], // KP multiply + 107: [0xFFAB], // KP add + 109: [0xFFAD], // KP subtract + 110: [0xFFAE], // KP decimal + 111: [0xFFAF], // KP divide + 112: [0xFFBE], // f1 + 113: [0xFFBF], // f2 + 114: [0xFFC0], // f3 + 115: [0xFFC1], // f4 + 116: [0xFFC2], // f5 + 117: [0xFFC3], // f6 + 118: [0xFFC4], // f7 + 119: [0xFFC5], // f8 + 120: [0xFFC6], // f9 + 121: [0xFFC7], // f10 + 122: [0xFFC8], // f11 + 123: [0xFFC9], // f12 + 144: [0xFF7F], // num lock + 145: [0xFF14], // scroll lock + 225: [0xFE03] // altgraph (iso_level3_shift) + }; + + /** + * Map of known JavaScript keyidentifiers which do not map to typable + * characters to their unshifted X11 keysym equivalents. + * @private + */ + var keyidentifier_keysym = { + "Again": [0xFF66], + "AllCandidates": [0xFF3D], + "Alphanumeric": [0xFF30], + "Alt": [0xFFE9, 0xFFE9, 0xFE03], + "Attn": [0xFD0E], + "AltGraph": [0xFE03], + "ArrowDown": [0xFF54], + "ArrowLeft": [0xFF51], + "ArrowRight": [0xFF53], + "ArrowUp": [0xFF52], + "Backspace": [0xFF08], + "CapsLock": [0xFFE5], + "Cancel": [0xFF69], + "Clear": [0xFF0B], + "Convert": [0xFF21], + "Copy": [0xFD15], + "Crsel": [0xFD1C], + "CrSel": [0xFD1C], + "CodeInput": [0xFF37], + "Compose": [0xFF20], + "Control": [0xFFE3, 0xFFE3, 0xFFE4], + "ContextMenu": [0xFF67], + "DeadGrave": [0xFE50], + "DeadAcute": [0xFE51], + "DeadCircumflex": [0xFE52], + "DeadTilde": [0xFE53], + "DeadMacron": [0xFE54], + "DeadBreve": [0xFE55], + "DeadAboveDot": [0xFE56], + "DeadUmlaut": [0xFE57], + "DeadAboveRing": [0xFE58], + "DeadDoubleacute": [0xFE59], + "DeadCaron": [0xFE5A], + "DeadCedilla": [0xFE5B], + "DeadOgonek": [0xFE5C], + "DeadIota": [0xFE5D], + "DeadVoicedSound": [0xFE5E], + "DeadSemivoicedSound": [0xFE5F], + "Delete": [0xFFFF], + "Down": [0xFF54], + "End": [0xFF57], + "Enter": [0xFF0D], + "EraseEof": [0xFD06], + "Escape": [0xFF1B], + "Execute": [0xFF62], + "Exsel": [0xFD1D], + "ExSel": [0xFD1D], + "F1": [0xFFBE], + "F2": [0xFFBF], + "F3": [0xFFC0], + "F4": [0xFFC1], + "F5": [0xFFC2], + "F6": [0xFFC3], + "F7": [0xFFC4], + "F8": [0xFFC5], + "F9": [0xFFC6], + "F10": [0xFFC7], + "F11": [0xFFC8], + "F12": [0xFFC9], + "F13": [0xFFCA], + "F14": [0xFFCB], + "F15": [0xFFCC], + "F16": [0xFFCD], + "F17": [0xFFCE], + "F18": [0xFFCF], + "F19": [0xFFD0], + "F20": [0xFFD1], + "F21": [0xFFD2], + "F22": [0xFFD3], + "F23": [0xFFD4], + "F24": [0xFFD5], + "Find": [0xFF68], + "GroupFirst": [0xFE0C], + "GroupLast": [0xFE0E], + "GroupNext": [0xFE08], + "GroupPrevious": [0xFE0A], + "FullWidth": null, + "HalfWidth": null, + "HangulMode": [0xFF31], + "Hankaku": [0xFF29], + "HanjaMode": [0xFF34], + "Help": [0xFF6A], + "Hiragana": [0xFF25], + "HiraganaKatakana": [0xFF27], + "Home": [0xFF50], + "Hyper": [0xFFED, 0xFFED, 0xFFEE], + "Insert": [0xFF63], + "JapaneseHiragana": [0xFF25], + "JapaneseKatakana": [0xFF26], + "JapaneseRomaji": [0xFF24], + "JunjaMode": [0xFF38], + "KanaMode": [0xFF2D], + "KanjiMode": [0xFF21], + "Katakana": [0xFF26], + "Left": [0xFF51], + "Meta": [0xFFE7, 0xFFE7, 0xFFE8], + "ModeChange": [0xFF7E], + "NumLock": [0xFF7F], + "PageDown": [0xFF56], + "PageUp": [0xFF55], + "Pause": [0xFF13], + "Play": [0xFD16], + "PreviousCandidate": [0xFF3E], + "PrintScreen": [0xFD1D], + "Redo": [0xFF66], + "Right": [0xFF53], + "RomanCharacters": null, + "Scroll": [0xFF14], + "Select": [0xFF60], + "Separator": [0xFFAC], + "Shift": [0xFFE1, 0xFFE1, 0xFFE2], + "SingleCandidate": [0xFF3C], + "Super": [0xFFEB, 0xFFEB, 0xFFEC], + "Tab": [0xFF09], + "Up": [0xFF52], + "Undo": [0xFF65], + "Win": [0xFFEB], + "Zenkaku": [0xFF28], + "ZenkakuHankaku": [0xFF2A] + }; + + const keysym_to_string = { + 0xFFFFFF: "VoidSymbol", + 0xFF08: "BackSpace", + 0xFF09: "Tab", + 0xFF0A: "Linefeed", + 0xFF0B: "Clear", + 0xFF0D: "Return", + 0xFF13: "Pause", + 0xFF14: "Scroll_Lock", + 0xFF15: "Sys_Req", + 0xFF1B: "Escape", + 0xFFFF: "Delete", + 0xFF20: "Multi_key", + 0xFF37: "Codeinput", + 0xFF3C: "SingleCandidate", + 0xFF3D: "MultipleCandidate", + 0xFF3E: "PreviousCandidate", + 0xFF21: "Kanji", + 0xFF22: "Muhenkan", + 0xFF23: "Henkan_Mode", + 0xFF23: "Henkan", + 0xFF24: "Romaji", + 0xFF25: "Hiragana", + 0xFF26: "Katakana", + 0xFF27: "Hiragana_Katakana", + 0xFF28: "Zenkaku", + 0xFF29: "Hankaku", + 0xFF2A: "Zenkaku_Hankaku", + 0xFF2B: "Touroku", + 0xFF2C: "Massyo", + 0xFF2D: "Kana_Lock", + 0xFF2E: "Kana_Shift", + 0xFF2F: "Eisu_Shift", + 0xFF30: "Eisu_toggle", + 0xFF37: "Kanji_Bangou", + 0xFF3D: "Zen_Koho", + 0xFF3E: "Mae_Koho", + 0xFF50: "Home", + 0xFF51: "Left", + 0xFF52: "Up", + 0xFF53: "Right", + 0xFF54: "Down", + 0xFF55: "Prior", + 0xFF55: "Page_Up", + 0xFF56: "Next", + 0xFF56: "Page_Down", + 0xFF57: "End", + 0xFF58: "Begin", + 0xFF60: "Select", + 0xFF61: "Print", + 0xFF62: "Execute", + 0xFF63: "Insert", + 0xFF65: "Undo", + 0xFF66: "Redo", + 0xFF67: "Menu", + 0xFF68: "Find", + 0xFF69: "Cancel", + 0xFF6A: "Help", + 0xFF6B: "Break", + 0xFF7E: "Mode_switch", + 0xFF7E: "script_switch", + 0xFF7F: "Num_Lock", + 0xFF80: "KP_Space", + 0xFF89: "KP_Tab", + 0xFF8D: "KP_Enter", + 0xFF91: "KP_F1", + 0xFF92: "KP_F2", + 0xFF93: "KP_F3", + 0xFF94: "KP_F4", + 0xFF95: "KP_Home", + 0xFF96: "KP_Left", + 0xFF97: "KP_Up", + 0xFF98: "KP_Right", + 0xFF99: "KP_Down", + 0xFF9A: "KP_Prior", + 0xFF9A: "KP_Page_Up", + 0xFF9B: "KP_Next", + 0xFF9B: "KP_Page_Down", + 0xFF9C: "KP_End", + 0xFF9D: "KP_Begin", + 0xFF9E: "KP_Insert", + 0xFF9F: "KP_Delete", + 0xFFBD: "KP_Equal", + 0xFFAA: "KP_Multiply", + 0xFFAB: "KP_Add", + 0xFFAC: "KP_Separator", + 0xFFAD: "KP_Subtract", + 0xFFAE: "KP_Decimal", + 0xFFAF: "KP_Divide", + 0xFFB0: "KP_0", + 0xFFB1: "KP_1", + 0xFFB2: "KP_2", + 0xFFB3: "KP_3", + 0xFFB4: "KP_4", + 0xFFB5: "KP_5", + 0xFFB6: "KP_6", + 0xFFB7: "KP_7", + 0xFFB8: "KP_8", + 0xFFB9: "KP_9", + 0xFFBE: "F1", + 0xFFBF: "F2", + 0xFFC0: "F3", + 0xFFC1: "F4", + 0xFFC2: "F5", + 0xFFC3: "F6", + 0xFFC4: "F7", + 0xFFC5: "F8", + 0xFFC6: "F9", + 0xFFC7: "F10", + 0xFFC8: "F11", + 0xFFC8: "L1", + 0xFFC9: "F12", + 0xFFC9: "L2", + 0xFFCA: "F13", + 0xFFCA: "L3", + 0xFFCB: "F14", + 0xFFCB: "L4", + 0xFFCC: "F15", + 0xFFCC: "L5", + 0xFFCD: "F16", + 0xFFCD: "L6", + 0xFFCE: "F17", + 0xFFCE: "L7", + 0xFFCF: "F18", + 0xFFCF: "L8", + 0xFFD0: "F19", + 0xFFD0: "L9", + 0xFFD1: "F20", + 0xFFD1: "L10", + 0xFFD2: "F21", + 0xFFD2: "R1", + 0xFFD3: "F22", + 0xFFD3: "R2", + 0xFFD4: "F23", + 0xFFD4: "R3", + 0xFFD5: "F24", + 0xFFD5: "R4", + 0xFFD6: "F25", + 0xFFD6: "R5", + 0xFFD7: "F26", + 0xFFD7: "R6", + 0xFFD8: "F27", + 0xFFD8: "R7", + 0xFFD9: "F28", + 0xFFD9: "R8", + 0xFFDA: "F29", + 0xFFDA: "R9", + 0xFFDB: "F30", + 0xFFDB: "R10", + 0xFFDC: "F31", + 0xFFDC: "R11", + 0xFFDD: "F32", + 0xFFDD: "R12", + 0xFFDE: "F33", + 0xFFDE: "R13", + 0xFFDF: "F34", + 0xFFDF: "R14", + 0xFFE0: "F35", + 0xFFE0: "R15", + 0xFFE1: "Shift_L", + 0xFFE2: "Shift_R", + 0xFFE3: "Control_L", + 0xFFE4: "Control_R", + 0xFFE5: "Caps_Lock", + 0xFFE6: "Shift_Lock", + 0xFFE7: "Meta_L", + 0xFFE8: "Meta_R", + 0xFFE9: "Alt_L", + 0xFFEA: "Alt_R", + 0xFFEB: "Super_L", + 0xFFEC: "Super_R", + 0xFFED: "Hyper_L", + 0xFFEE: "Hyper_R", + 0xFE01: "ISO_Lock", + 0xFE02: "ISO_Level2_Latch", + 0xFE03: "ISO_Level3_Shift", + 0xFE04: "ISO_Level3_Latch", + 0xFE05: "ISO_Level3_Lock", + 0xFE11: "ISO_Level5_Shift", + 0xFE12: "ISO_Level5_Latch", + 0xFE13: "ISO_Level5_Lock", + 0xFF7E: "ISO_Group_Shift", + 0xFE06: "ISO_Group_Latch", + 0xFE07: "ISO_Group_Lock", + 0xFE08: "ISO_Next_Group", + 0xFE09: "ISO_Next_Group_Lock", + 0xFE0A: "ISO_Prev_Group", + 0xFE0B: "ISO_Prev_Group_Lock", + 0xFE0C: "ISO_First_Group", + 0xFE0D: "ISO_First_Group_Lock", + 0xFE0E: "ISO_Last_Group", + 0xFE0F: "ISO_Last_Group_Lock", + 0xFE20: "ISO_Left_Tab", + 0xFE21: "ISO_Move_Line_Up", + 0xFE22: "ISO_Move_Line_Down", + 0xFE23: "ISO_Partial_Line_Up", + 0xFE24: "ISO_Partial_Line_Down", + 0xFE25: "ISO_Partial_Space_Left", + 0xFE26: "ISO_Partial_Space_Right", + 0xFE27: "ISO_Set_Margin_Left", + 0xFE28: "ISO_Set_Margin_Right", + 0xFE29: "ISO_Release_Margin_Left", + 0xFE2A: "ISO_Release_Margin_Right", + 0xFE2B: "ISO_Release_Both_Margins", + 0xFE2C: "ISO_Fast_Cursor_Left", + 0xFE2D: "ISO_Fast_Cursor_Right", + 0xFE2E: "ISO_Fast_Cursor_Up", + 0xFE2F: "ISO_Fast_Cursor_Down", + 0xFE30: "ISO_Continuous_Underline", + 0xFE31: "ISO_Discontinuous_Underline", + 0xFE32: "ISO_Emphasize", + 0xFE33: "ISO_Center_Object", + 0xFE34: "ISO_Enter", + 0xFE50: "dead_grave", + 0xFE51: "dead_acute", + 0xFE52: "dead_circumflex", + 0xFE53: "dead_tilde", + 0xFE53: "dead_perispomeni", + 0xFE54: "dead_macron", + 0xFE55: "dead_breve", + 0xFE56: "dead_abovedot", + 0xFE57: "dead_diaeresis", + 0xFE58: "dead_abovering", + 0xFE59: "dead_doubleacute", + 0xFE5A: "dead_caron", + 0xFE5B: "dead_cedilla", + 0xFE5C: "dead_ogonek", + 0xFE5D: "dead_iota", + 0xFE5E: "dead_voiced_sound", + 0xFE5F: "dead_semivoiced_sound", + 0xFE60: "dead_belowdot", + 0xFE61: "dead_hook", + 0xFE62: "dead_horn", + 0xFE63: "dead_stroke", + 0xFE64: "dead_abovecomma", + 0xFE64: "dead_psili", + 0xFE65: "dead_abovereversedcomma", + 0xFE65: "dead_dasia", + 0xFE66: "dead_doublegrave", + 0xFE67: "dead_belowring", + 0xFE68: "dead_belowmacron", + 0xFE69: "dead_belowcircumflex", + 0xFE6A: "dead_belowtilde", + 0xFE6B: "dead_belowbreve", + 0xFE6C: "dead_belowdiaeresis", + 0xFE6D: "dead_invertedbreve", + 0xFE6E: "dead_belowcomma", + 0xFE6F: "dead_currency", + 0xFE90: "dead_lowline", + 0xFE91: "dead_aboveverticalline", + 0xFE92: "dead_belowverticalline", + 0xFE93: "dead_longsolidusoverlay", + 0xFE80: "dead_a", + 0xFE81: "dead_A", + 0xFE82: "dead_e", + 0xFE83: "dead_E", + 0xFE84: "dead_i", + 0xFE85: "dead_I", + 0xFE86: "dead_o", + 0xFE87: "dead_O", + 0xFE88: "dead_u", + 0xFE89: "dead_U", + 0xFE8A: "dead_small_schwa", + 0xFE8B: "dead_capital_schwa", + 0xFE8C: "dead_greek", + 0xFED0: "First_Virtual_Screen", + 0xFED1: "Prev_Virtual_Screen", + 0xFED2: "Next_Virtual_Screen", + 0xFED4: "Last_Virtual_Screen", + 0xFED5: "Terminate_Server", + 0xFE70: "AccessX_Enable", + 0xFE71: "AccessX_Feedback_Enable", + 0xFE72: "RepeatKeys_Enable", + 0xFE73: "SlowKeys_Enable", + 0xFE74: "BounceKeys_Enable", + 0xFE75: "StickyKeys_Enable", + 0xFE76: "MouseKeys_Enable", + 0xFE77: "MouseKeys_Accel_Enable", + 0xFE78: "Overlay1_Enable", + 0xFE79: "Overlay2_Enable", + 0xFE7A: "AudibleBell_Enable", + 0xFEE0: "Pointer_Left", + 0xFEE1: "Pointer_Right", + 0xFEE2: "Pointer_Up", + 0xFEE3: "Pointer_Down", + 0xFEE4: "Pointer_UpLeft", + 0xFEE5: "Pointer_UpRight", + 0xFEE6: "Pointer_DownLeft", + 0xFEE7: "Pointer_DownRight", + 0xFEE8: "Pointer_Button_Dflt", + 0xFEE9: "Pointer_Button1", + 0xFEEA: "Pointer_Button2", + 0xFEEB: "Pointer_Button3", + 0xFEEC: "Pointer_Button4", + 0xFEED: "Pointer_Button5", + 0xFEEE: "Pointer_DblClick_Dflt", + 0xFEEF: "Pointer_DblClick1", + 0xFEF0: "Pointer_DblClick2", + 0xFEF1: "Pointer_DblClick3", + 0xFEF2: "Pointer_DblClick4", + 0xFEF3: "Pointer_DblClick5", + 0xFEF4: "Pointer_Drag_Dflt", + 0xFEF5: "Pointer_Drag1", + 0xFEF6: "Pointer_Drag2", + 0xFEF7: "Pointer_Drag3", + 0xFEF8: "Pointer_Drag4", + 0xFEFD: "Pointer_Drag5", + 0xFEF9: "Pointer_EnableKeys", + 0xFEFA: "Pointer_Accelerate", + 0xFEFB: "Pointer_DfltBtnNext", + 0xFEFC: "Pointer_DfltBtnPrev", + 0xFEA0: "ch", + 0xFEA1: "Ch", + 0xFEA2: "CH", + 0xFEA3: "c_h", + 0xFEA4: "C_h", + 0xFEA5: "C_H", + 0xFD01: "3270_Duplicate", + 0xFD02: "3270_FieldMark", + 0xFD03: "3270_Right2", + 0xFD04: "3270_Left2", + 0xFD05: "3270_BackTab", + 0xFD06: "3270_EraseEOF", + 0xFD07: "3270_EraseInput", + 0xFD08: "3270_Reset", + 0xFD09: "3270_Quit", + 0xFD0A: "3270_PA1", + 0xFD0B: "3270_PA2", + 0xFD0C: "3270_PA3", + 0xFD0D: "3270_Test", + 0xFD0E: "3270_Attn", + 0xFD0F: "3270_CursorBlink", + 0xFD10: "3270_AltCursor", + 0xFD11: "3270_KeyClick", + 0xFD12: "3270_Jump", + 0xFD13: "3270_Ident", + 0xFD14: "3270_Rule", + 0xFD15: "3270_Copy", + 0xFD16: "3270_Play", + 0xFD17: "3270_Setup", + 0xFD18: "3270_Record", + 0xFD19: "3270_ChangeScreen", + 0xFD1A: "3270_DeleteWord", + 0xFD1B: "3270_ExSelect", + 0xFD1C: "3270_CursorSelect", + 0xFD1D: "3270_PrintScreen", + 0xFD1E: "3270_Enter", + 0x0020: "space", + 0x0021: "exclam", + 0x0022: "quotedbl", + 0x0023: "numbersign", + 0x0024: "dollar", + 0x0025: "percent", + 0x0026: "ampersand", + 0x0027: "apostrophe", + 0x0027: "quoteright", + 0x0028: "parenleft", + 0x0029: "parenright", + 0x002A: "asterisk", + 0x002B: "plus", + 0x002C: "comma", + 0x002D: "minus", + 0x002E: "period", + 0x002F: "slash", + 0x0030: "0", + 0x0031: "1", + 0x0032: "2", + 0x0033: "3", + 0x0034: "4", + 0x0035: "5", + 0x0036: "6", + 0x0037: "7", + 0x0038: "8", + 0x0039: "9", + 0x003A: "colon", + 0x003B: "semicolon", + 0x003C: "less", + 0x003D: "equal", + 0x003E: "greater", + 0x003F: "question", + 0x0040: "at", + 0x0041: "A", + 0x0042: "B", + 0x0043: "C", + 0x0044: "D", + 0x0045: "E", + 0x0046: "F", + 0x0047: "G", + 0x0048: "H", + 0x0049: "I", + 0x004A: "J", + 0x004B: "K", + 0x004C: "L", + 0x004D: "M", + 0x004E: "N", + 0x004F: "O", + 0x0050: "P", + 0x0051: "Q", + 0x0052: "R", + 0x0053: "S", + 0x0054: "T", + 0x0055: "U", + 0x0056: "V", + 0x0057: "W", + 0x0058: "X", + 0x0059: "Y", + 0x005A: "Z", + 0x005B: "bracketleft", + 0x005C: "backslash", + 0x005D: "bracketright", + 0x005E: "asciicircum", + 0x005F: "underscore", + 0x0060: "grave", + 0x0060: "quoteleft", + 0x0061: "a", + 0x0062: "b", + 0x0063: "c", + 0x0064: "d", + 0x0065: "e", + 0x0066: "f", + 0x0067: "g", + 0x0068: "h", + 0x0069: "i", + 0x006A: "j", + 0x006B: "k", + 0x006C: "l", + 0x006D: "m", + 0x006E: "n", + 0x006F: "o", + 0x0070: "p", + 0x0071: "q", + 0x0072: "r", + 0x0073: "s", + 0x0074: "t", + 0x0075: "u", + 0x0076: "v", + 0x0077: "w", + 0x0078: "x", + 0x0079: "y", + 0x007A: "z", + 0x007B: "braceleft", + 0x007C: "bar", + 0x007D: "braceright", + 0x007E: "asciitilde", + 0x00A0: "nobreakspace", + 0x00A1: "exclamdown", + 0x00A2: "cent", + 0x00A3: "sterling", + 0x00A4: "currency", + 0x00A5: "yen", + 0x00A6: "brokenbar", + 0x00A7: "section", + 0x00A8: "diaeresis", + 0x00A9: "copyright", + 0x00AA: "ordfeminine", + 0x00AB: "guillemotleft", + 0x00AC: "notsign", + 0x00AD: "hyphen", + 0x00AE: "registered", + 0x00AF: "macron", + 0x00B0: "degree", + 0x00B1: "plusminus", + 0x00B2: "twosuperior", + 0x00B3: "threesuperior", + 0x00B4: "acute", + 0x00B5: "mu", + 0x00B6: "paragraph", + 0x00B7: "periodcentered", + 0x00B8: "cedilla", + 0x00B9: "onesuperior", + 0x00BA: "masculine", + 0x00BB: "guillemotright", + 0x00BC: "onequarter", + 0x00BD: "onehalf", + 0x00BE: "threequarters", + 0x00BF: "questiondown", + 0x00C0: "Agrave", + 0x00C1: "Aacute", + 0x00C2: "Acircumflex", + 0x00C3: "Atilde", + 0x00C4: "Adiaeresis", + 0x00C5: "Aring", + 0x00C6: "AE", + 0x00C7: "Ccedilla", + 0x00C8: "Egrave", + 0x00C9: "Eacute", + 0x00CA: "Ecircumflex", + 0x00CB: "Ediaeresis", + 0x00CC: "Igrave", + 0x00CD: "Iacute", + 0x00CE: "Icircumflex", + 0x00CF: "Idiaeresis", + 0x00D0: "ETH", + 0x00D0: "Eth", + 0x00D1: "Ntilde", + 0x00D2: "Ograve", + 0x00D3: "Oacute", + 0x00D4: "Ocircumflex", + 0x00D5: "Otilde", + 0x00D6: "Odiaeresis", + 0x00D7: "multiply", + 0x00D8: "Oslash", + 0x00D8: "Ooblique", + 0x00D9: "Ugrave", + 0x00DA: "Uacute", + 0x00DB: "Ucircumflex", + 0x00DC: "Udiaeresis", + 0x00DD: "Yacute", + 0x00DE: "THORN", + 0x00DE: "Thorn", + 0x00DF: "ssharp", + 0x00E0: "agrave", + 0x00E1: "aacute", + 0x00E2: "acircumflex", + 0x00E3: "atilde", + 0x00E4: "adiaeresis", + 0x00E5: "aring", + 0x00E6: "ae", + 0x00E7: "ccedilla", + 0x00E8: "egrave", + 0x00E9: "eacute", + 0x00EA: "ecircumflex", + 0x00EB: "ediaeresis", + 0x00EC: "igrave", + 0x00ED: "iacute", + 0x00EE: "icircumflex", + 0x00EF: "idiaeresis", + 0x00F0: "eth", + 0x00F1: "ntilde", + 0x00F2: "ograve", + 0x00F3: "oacute", + 0x00F4: "ocircumflex", + 0x00F5: "otilde", + 0x00F6: "odiaeresis", + 0x00F7: "division", + 0x00F8: "oslash", + 0x00F8: "ooblique", + 0x00F9: "ugrave", + 0x00FA: "uacute", + 0x00FB: "ucircumflex", + 0x00FC: "udiaeresis", + 0x00FD: "yacute", + 0x00FE: "thorn", + 0x00FF: "ydiaeresis", + 0x01A1: "Aogonek", + 0x01A2: "breve", + 0x01A3: "Lstroke", + 0x01A5: "Lcaron", + 0x01A6: "Sacute", + 0x01A9: "Scaron", + 0x01AA: "Scedilla", + 0x01AB: "Tcaron", + 0x01AC: "Zacute", + 0x01AE: "Zcaron", + 0x01AF: "Zabovedot", + 0x01B1: "aogonek", + 0x01B2: "ogonek", + 0x01B3: "lstroke", + 0x01B5: "lcaron", + 0x01B6: "sacute", + 0x01B7: "caron", + 0x01B9: "scaron", + 0x01BA: "scedilla", + 0x01BB: "tcaron", + 0x01BC: "zacute", + 0x01BD: "doubleacute", + 0x01BE: "zcaron", + 0x01BF: "zabovedot", + 0x01C0: "Racute", + 0x01C3: "Abreve", + 0x01C5: "Lacute", + 0x01C6: "Cacute", + 0x01C8: "Ccaron", + 0x01CA: "Eogonek", + 0x01CC: "Ecaron", + 0x01CF: "Dcaron", + 0x01D0: "Dstroke", + 0x01D1: "Nacute", + 0x01D2: "Ncaron", + 0x01D5: "Odoubleacute", + 0x01D8: "Rcaron", + 0x01D9: "Uring", + 0x01DB: "Udoubleacute", + 0x01DE: "Tcedilla", + 0x01E0: "racute", + 0x01E3: "abreve", + 0x01E5: "lacute", + 0x01E6: "cacute", + 0x01E8: "ccaron", + 0x01EA: "eogonek", + 0x01EC: "ecaron", + 0x01EF: "dcaron", + 0x01F0: "dstroke", + 0x01F1: "nacute", + 0x01F2: "ncaron", + 0x01F5: "odoubleacute", + 0x01F8: "rcaron", + 0x01F9: "uring", + 0x01FB: "udoubleacute", + 0x01FE: "tcedilla", + 0x01FF: "abovedot", + 0x02A1: "Hstroke", + 0x02A6: "Hcircumflex", + 0x02A9: "Iabovedot", + 0x02AB: "Gbreve", + 0x02AC: "Jcircumflex", + 0x02B1: "hstroke", + 0x02B6: "hcircumflex", + 0x02B9: "idotless", + 0x02BB: "gbreve", + 0x02BC: "jcircumflex", + 0x02C5: "Cabovedot", + 0x02C6: "Ccircumflex", + 0x02D5: "Gabovedot", + 0x02D8: "Gcircumflex", + 0x02DD: "Ubreve", + 0x02DE: "Scircumflex", + 0x02E5: "cabovedot", + 0x02E6: "ccircumflex", + 0x02F5: "gabovedot", + 0x02F8: "gcircumflex", + 0x02FD: "ubreve", + 0x02FE: "scircumflex", + 0x03A2: "kra", + 0x03A2: "kappa", + 0x03A3: "Rcedilla", + 0x03A5: "Itilde", + 0x03A6: "Lcedilla", + 0x03AA: "Emacron", + 0x03AB: "Gcedilla", + 0x03AC: "Tslash", + 0x03B3: "rcedilla", + 0x03B5: "itilde", + 0x03B6: "lcedilla", + 0x03BA: "emacron", + 0x03BB: "gcedilla", + 0x03BC: "tslash", + 0x03BD: "ENG", + 0x03BF: "eng", + 0x03C0: "Amacron", + 0x03C7: "Iogonek", + 0x03CC: "Eabovedot", + 0x03CF: "Imacron", + 0x03D1: "Ncedilla", + 0x03D2: "Omacron", + 0x03D3: "Kcedilla", + 0x03D9: "Uogonek", + 0x03DD: "Utilde", + 0x03DE: "Umacron", + 0x03E0: "amacron", + 0x03E7: "iogonek", + 0x03EC: "eabovedot", + 0x03EF: "imacron", + 0x03F1: "ncedilla", + 0x03F2: "omacron", + 0x03F3: "kcedilla", + 0x03F9: "uogonek", + 0x03FD: "utilde", + 0x03FE: "umacron", + 0x1000174: "Wcircumflex", + 0x1000175: "wcircumflex", + 0x1000176: "Ycircumflex", + 0x1000177: "ycircumflex", + 0x1001E02: "Babovedot", + 0x1001E03: "babovedot", + 0x1001E0A: "Dabovedot", + 0x1001E0B: "dabovedot", + 0x1001E1E: "Fabovedot", + 0x1001E1F: "fabovedot", + 0x1001E40: "Mabovedot", + 0x1001E41: "mabovedot", + 0x1001E56: "Pabovedot", + 0x1001E57: "pabovedot", + 0x1001E60: "Sabovedot", + 0x1001E61: "sabovedot", + 0x1001E6A: "Tabovedot", + 0x1001E6B: "tabovedot", + 0x1001E80: "Wgrave", + 0x1001E81: "wgrave", + 0x1001E82: "Wacute", + 0x1001E83: "wacute", + 0x1001E84: "Wdiaeresis", + 0x1001E85: "wdiaeresis", + 0x1001EF2: "Ygrave", + 0x1001EF3: "ygrave", + 0x13BC: "OE", + 0x13BD: "oe", + 0x13BE: "Ydiaeresis", + 0x047E: "overline", + 0x04A1: "kana_fullstop", + 0x04A2: "kana_openingbracket", + 0x04A3: "kana_closingbracket", + 0x04A4: "kana_comma", + 0x04A5: "kana_conjunctive", + 0x04A5: "kana_middledot", + 0x04A6: "kana_WO", + 0x04A7: "kana_a", + 0x04A8: "kana_i", + 0x04A9: "kana_u", + 0x04AA: "kana_e", + 0x04AB: "kana_o", + 0x04AC: "kana_ya", + 0x04AD: "kana_yu", + 0x04AE: "kana_yo", + 0x04AF: "kana_tsu", + 0x04AF: "kana_tu", + 0x04B0: "prolongedsound", + 0x04B1: "kana_A", + 0x04B2: "kana_I", + 0x04B3: "kana_U", + 0x04B4: "kana_E", + 0x04B5: "kana_O", + 0x04B6: "kana_KA", + 0x04B7: "kana_KI", + 0x04B8: "kana_KU", + 0x04B9: "kana_KE", + 0x04BA: "kana_KO", + 0x04BB: "kana_SA", + 0x04BC: "kana_SHI", + 0x04BD: "kana_SU", + 0x04BE: "kana_SE", + 0x04BF: "kana_SO", + 0x04C0: "kana_TA", + 0x04C1: "kana_CHI", + 0x04C1: "kana_TI", + 0x04C2: "kana_TSU", + 0x04C2: "kana_TU", + 0x04C3: "kana_TE", + 0x04C4: "kana_TO", + 0x04C5: "kana_NA", + 0x04C6: "kana_NI", + 0x04C7: "kana_NU", + 0x04C8: "kana_NE", + 0x04C9: "kana_NO", + 0x04CA: "kana_HA", + 0x04CB: "kana_HI", + 0x04CC: "kana_FU", + 0x04CC: "kana_HU", + 0x04CD: "kana_HE", + 0x04CE: "kana_HO", + 0x04CF: "kana_MA", + 0x04D0: "kana_MI", + 0x04D1: "kana_MU", + 0x04D2: "kana_ME", + 0x04D3: "kana_MO", + 0x04D4: "kana_YA", + 0x04D5: "kana_YU", + 0x04D6: "kana_YO", + 0x04D7: "kana_RA", + 0x04D8: "kana_RI", + 0x04D9: "kana_RU", + 0x04DA: "kana_RE", + 0x04DB: "kana_RO", + 0x04DC: "kana_WA", + 0x04DD: "kana_N", + 0x04DE: "voicedsound", + 0x04DF: "semivoicedsound", + 0xFF7E: "kana_switch", + 0x10006F0: "Farsi_0", + 0x10006F1: "Farsi_1", + 0x10006F2: "Farsi_2", + 0x10006F3: "Farsi_3", + 0x10006F4: "Farsi_4", + 0x10006F5: "Farsi_5", + 0x10006F6: "Farsi_6", + 0x10006F7: "Farsi_7", + 0x10006F8: "Farsi_8", + 0x10006F9: "Farsi_9", + 0x100066A: "Arabic_percent", + 0x1000670: "Arabic_superscript_alef", + 0x1000679: "Arabic_tteh", + 0x100067E: "Arabic_peh", + 0x1000686: "Arabic_tcheh", + 0x1000688: "Arabic_ddal", + 0x1000691: "Arabic_rreh", + 0x05AC: "Arabic_comma", + 0x10006D4: "Arabic_fullstop", + 0x1000660: "Arabic_0", + 0x1000661: "Arabic_1", + 0x1000662: "Arabic_2", + 0x1000663: "Arabic_3", + 0x1000664: "Arabic_4", + 0x1000665: "Arabic_5", + 0x1000666: "Arabic_6", + 0x1000667: "Arabic_7", + 0x1000668: "Arabic_8", + 0x1000669: "Arabic_9", + 0x05BB: "Arabic_semicolon", + 0x05BF: "Arabic_question_mark", + 0x05C1: "Arabic_hamza", + 0x05C2: "Arabic_maddaonalef", + 0x05C3: "Arabic_hamzaonalef", + 0x05C4: "Arabic_hamzaonwaw", + 0x05C5: "Arabic_hamzaunderalef", + 0x05C6: "Arabic_hamzaonyeh", + 0x05C7: "Arabic_alef", + 0x05C8: "Arabic_beh", + 0x05C9: "Arabic_tehmarbuta", + 0x05CA: "Arabic_teh", + 0x05CB: "Arabic_theh", + 0x05CC: "Arabic_jeem", + 0x05CD: "Arabic_hah", + 0x05CE: "Arabic_khah", + 0x05CF: "Arabic_dal", + 0x05D0: "Arabic_thal", + 0x05D1: "Arabic_ra", + 0x05D2: "Arabic_zain", + 0x05D3: "Arabic_seen", + 0x05D4: "Arabic_sheen", + 0x05D5: "Arabic_sad", + 0x05D6: "Arabic_dad", + 0x05D7: "Arabic_tah", + 0x05D8: "Arabic_zah", + 0x05D9: "Arabic_ain", + 0x05DA: "Arabic_ghain", + 0x05E0: "Arabic_tatweel", + 0x05E1: "Arabic_feh", + 0x05E2: "Arabic_qaf", + 0x05E3: "Arabic_kaf", + 0x05E4: "Arabic_lam", + 0x05E5: "Arabic_meem", + 0x05E6: "Arabic_noon", + 0x05E7: "Arabic_ha", + 0x05E7: "Arabic_heh", + 0x05E8: "Arabic_waw", + 0x05E9: "Arabic_alefmaksura", + 0x05EA: "Arabic_yeh", + 0x05EB: "Arabic_fathatan", + 0x05EC: "Arabic_dammatan", + 0x05ED: "Arabic_kasratan", + 0x05EE: "Arabic_fatha", + 0x05EF: "Arabic_damma", + 0x05F0: "Arabic_kasra", + 0x05F1: "Arabic_shadda", + 0x05F2: "Arabic_sukun", + 0x1000653: "Arabic_madda_above", + 0x1000654: "Arabic_hamza_above", + 0x1000655: "Arabic_hamza_below", + 0x1000698: "Arabic_jeh", + 0x10006A4: "Arabic_veh", + 0x10006A9: "Arabic_keheh", + 0x10006AF: "Arabic_gaf", + 0x10006BA: "Arabic_noon_ghunna", + 0x10006BE: "Arabic_heh_doachashmee", + 0x10006CC: "Farsi_yeh", + 0x10006CC: "Arabic_farsi_yeh", + 0x10006D2: "Arabic_yeh_baree", + 0x10006C1: "Arabic_heh_goal", + 0xFF7E: "Arabic_switch", + 0x1000492: "Cyrillic_GHE_bar", + 0x1000493: "Cyrillic_ghe_bar", + 0x1000496: "Cyrillic_ZHE_descender", + 0x1000497: "Cyrillic_zhe_descender", + 0x100049A: "Cyrillic_KA_descender", + 0x100049B: "Cyrillic_ka_descender", + 0x100049C: "Cyrillic_KA_vertstroke", + 0x100049D: "Cyrillic_ka_vertstroke", + 0x10004A2: "Cyrillic_EN_descender", + 0x10004A3: "Cyrillic_en_descender", + 0x10004AE: "Cyrillic_U_straight", + 0x10004AF: "Cyrillic_u_straight", + 0x10004B0: "Cyrillic_U_straight_bar", + 0x10004B1: "Cyrillic_u_straight_bar", + 0x10004B2: "Cyrillic_HA_descender", + 0x10004B3: "Cyrillic_ha_descender", + 0x10004B6: "Cyrillic_CHE_descender", + 0x10004B7: "Cyrillic_che_descender", + 0x10004B8: "Cyrillic_CHE_vertstroke", + 0x10004B9: "Cyrillic_che_vertstroke", + 0x10004BA: "Cyrillic_SHHA", + 0x10004BB: "Cyrillic_shha", + 0x10004D8: "Cyrillic_SCHWA", + 0x10004D9: "Cyrillic_schwa", + 0x10004E2: "Cyrillic_I_macron", + 0x10004E3: "Cyrillic_i_macron", + 0x10004E8: "Cyrillic_O_bar", + 0x10004E9: "Cyrillic_o_bar", + 0x10004EE: "Cyrillic_U_macron", + 0x10004EF: "Cyrillic_u_macron", + 0x06A1: "Serbian_dje", + 0x06A2: "Macedonia_gje", + 0x06A3: "Cyrillic_io", + 0x06A4: "Ukrainian_ie", + 0x06A4: "Ukranian_je", + 0x06A5: "Macedonia_dse", + 0x06A6: "Ukrainian_i", + 0x06A6: "Ukranian_i", + 0x06A7: "Ukrainian_yi", + 0x06A7: "Ukranian_yi", + 0x06A8: "Cyrillic_je", + 0x06A8: "Serbian_je", + 0x06A9: "Cyrillic_lje", + 0x06A9: "Serbian_lje", + 0x06AA: "Cyrillic_nje", + 0x06AA: "Serbian_nje", + 0x06AB: "Serbian_tshe", + 0x06AC: "Macedonia_kje", + 0x06AD: "Ukrainian_ghe_with_upturn", + 0x06AE: "Byelorussian_shortu", + 0x06AF: "Cyrillic_dzhe", + 0x06AF: "Serbian_dze", + 0x06B0: "numerosign", + 0x06B1: "Serbian_DJE", + 0x06B2: "Macedonia_GJE", + 0x06B3: "Cyrillic_IO", + 0x06B4: "Ukrainian_IE", + 0x06B4: "Ukranian_JE", + 0x06B5: "Macedonia_DSE", + 0x06B6: "Ukrainian_I", + 0x06B6: "Ukranian_I", + 0x06B7: "Ukrainian_YI", + 0x06B7: "Ukranian_YI", + 0x06B8: "Cyrillic_JE", + 0x06B8: "Serbian_JE", + 0x06B9: "Cyrillic_LJE", + 0x06B9: "Serbian_LJE", + 0x06BA: "Cyrillic_NJE", + 0x06BA: "Serbian_NJE", + 0x06BB: "Serbian_TSHE", + 0x06BC: "Macedonia_KJE", + 0x06BD: "Ukrainian_GHE_WITH_UPTURN", + 0x06BE: "Byelorussian_SHORTU", + 0x06BF: "Cyrillic_DZHE", + 0x06BF: "Serbian_DZE", + 0x06C0: "Cyrillic_yu", + 0x06C1: "Cyrillic_a", + 0x06C2: "Cyrillic_be", + 0x06C3: "Cyrillic_tse", + 0x06C4: "Cyrillic_de", + 0x06C5: "Cyrillic_ie", + 0x06C6: "Cyrillic_ef", + 0x06C7: "Cyrillic_ghe", + 0x06C8: "Cyrillic_ha", + 0x06C9: "Cyrillic_i", + 0x06CA: "Cyrillic_shorti", + 0x06CB: "Cyrillic_ka", + 0x06CC: "Cyrillic_el", + 0x06CD: "Cyrillic_em", + 0x06CE: "Cyrillic_en", + 0x06CF: "Cyrillic_o", + 0x06D0: "Cyrillic_pe", + 0x06D1: "Cyrillic_ya", + 0x06D2: "Cyrillic_er", + 0x06D3: "Cyrillic_es", + 0x06D4: "Cyrillic_te", + 0x06D5: "Cyrillic_u", + 0x06D6: "Cyrillic_zhe", + 0x06D7: "Cyrillic_ve", + 0x06D8: "Cyrillic_softsign", + 0x06D9: "Cyrillic_yeru", + 0x06DA: "Cyrillic_ze", + 0x06DB: "Cyrillic_sha", + 0x06DC: "Cyrillic_e", + 0x06DD: "Cyrillic_shcha", + 0x06DE: "Cyrillic_che", + 0x06DF: "Cyrillic_hardsign", + 0x06E0: "Cyrillic_YU", + 0x06E1: "Cyrillic_A", + 0x06E2: "Cyrillic_BE", + 0x06E3: "Cyrillic_TSE", + 0x06E4: "Cyrillic_DE", + 0x06E5: "Cyrillic_IE", + 0x06E6: "Cyrillic_EF", + 0x06E7: "Cyrillic_GHE", + 0x06E8: "Cyrillic_HA", + 0x06E9: "Cyrillic_I", + 0x06EA: "Cyrillic_SHORTI", + 0x06EB: "Cyrillic_KA", + 0x06EC: "Cyrillic_EL", + 0x06ED: "Cyrillic_EM", + 0x06EE: "Cyrillic_EN", + 0x06EF: "Cyrillic_O", + 0x06F0: "Cyrillic_PE", + 0x06F1: "Cyrillic_YA", + 0x06F2: "Cyrillic_ER", + 0x06F3: "Cyrillic_ES", + 0x06F4: "Cyrillic_TE", + 0x06F5: "Cyrillic_U", + 0x06F6: "Cyrillic_ZHE", + 0x06F7: "Cyrillic_VE", + 0x06F8: "Cyrillic_SOFTSIGN", + 0x06F9: "Cyrillic_YERU", + 0x06FA: "Cyrillic_ZE", + 0x06FB: "Cyrillic_SHA", + 0x06FC: "Cyrillic_E", + 0x06FD: "Cyrillic_SHCHA", + 0x06FE: "Cyrillic_CHE", + 0x06FF: "Cyrillic_HARDSIGN", + 0x07A1: "Greek_ALPHAaccent", + 0x07A2: "Greek_EPSILONaccent", + 0x07A3: "Greek_ETAaccent", + 0x07A4: "Greek_IOTAaccent", + 0x07A5: "Greek_IOTAdieresis", + 0x07A5: "Greek_IOTAdiaeresis", + 0x07A7: "Greek_OMICRONaccent", + 0x07A8: "Greek_UPSILONaccent", + 0x07A9: "Greek_UPSILONdieresis", + 0x07AB: "Greek_OMEGAaccent", + 0x07AE: "Greek_accentdieresis", + 0x07AF: "Greek_horizbar", + 0x07B1: "Greek_alphaaccent", + 0x07B2: "Greek_epsilonaccent", + 0x07B3: "Greek_etaaccent", + 0x07B4: "Greek_iotaaccent", + 0x07B5: "Greek_iotadieresis", + 0x07B6: "Greek_iotaaccentdieresis", + 0x07B7: "Greek_omicronaccent", + 0x07B8: "Greek_upsilonaccent", + 0x07B9: "Greek_upsilondieresis", + 0x07BA: "Greek_upsilonaccentdieresis", + 0x07BB: "Greek_omegaaccent", + 0x07C1: "Greek_ALPHA", + 0x07C2: "Greek_BETA", + 0x07C3: "Greek_GAMMA", + 0x07C4: "Greek_DELTA", + 0x07C5: "Greek_EPSILON", + 0x07C6: "Greek_ZETA", + 0x07C7: "Greek_ETA", + 0x07C8: "Greek_THETA", + 0x07C9: "Greek_IOTA", + 0x07CA: "Greek_KAPPA", + 0x07CB: "Greek_LAMDA", + 0x07CB: "Greek_LAMBDA", + 0x07CC: "Greek_MU", + 0x07CD: "Greek_NU", + 0x07CE: "Greek_XI", + 0x07CF: "Greek_OMICRON", + 0x07D0: "Greek_PI", + 0x07D1: "Greek_RHO", + 0x07D2: "Greek_SIGMA", + 0x07D4: "Greek_TAU", + 0x07D5: "Greek_UPSILON", + 0x07D6: "Greek_PHI", + 0x07D7: "Greek_CHI", + 0x07D8: "Greek_PSI", + 0x07D9: "Greek_OMEGA", + 0x07E1: "Greek_alpha", + 0x07E2: "Greek_beta", + 0x07E3: "Greek_gamma", + 0x07E4: "Greek_delta", + 0x07E5: "Greek_epsilon", + 0x07E6: "Greek_zeta", + 0x07E7: "Greek_eta", + 0x07E8: "Greek_theta", + 0x07E9: "Greek_iota", + 0x07EA: "Greek_kappa", + 0x07EB: "Greek_lamda", + 0x07EB: "Greek_lambda", + 0x07EC: "Greek_mu", + 0x07ED: "Greek_nu", + 0x07EE: "Greek_xi", + 0x07EF: "Greek_omicron", + 0x07F0: "Greek_pi", + 0x07F1: "Greek_rho", + 0x07F2: "Greek_sigma", + 0x07F3: "Greek_finalsmallsigma", + 0x07F4: "Greek_tau", + 0x07F5: "Greek_upsilon", + 0x07F6: "Greek_phi", + 0x07F7: "Greek_chi", + 0x07F8: "Greek_psi", + 0x07F9: "Greek_omega", + 0xFF7E: "Greek_switch", + 0x08A1: "leftradical", + 0x08A2: "topleftradical", + 0x08A3: "horizconnector", + 0x08A4: "topintegral", + 0x08A5: "botintegral", + 0x08A6: "vertconnector", + 0x08A7: "topleftsqbracket", + 0x08A8: "botleftsqbracket", + 0x08A9: "toprightsqbracket", + 0x08AA: "botrightsqbracket", + 0x08AB: "topleftparens", + 0x08AC: "botleftparens", + 0x08AD: "toprightparens", + 0x08AE: "botrightparens", + 0x08AF: "leftmiddlecurlybrace", + 0x08B0: "rightmiddlecurlybrace", + 0x08B1: "topleftsummation", + 0x08B2: "botleftsummation", + 0x08B3: "topvertsummationconnector", + 0x08B4: "botvertsummationconnector", + 0x08B5: "toprightsummation", + 0x08B6: "botrightsummation", + 0x08B7: "rightmiddlesummation", + 0x08BC: "lessthanequal", + 0x08BD: "notequal", + 0x08BE: "greaterthanequal", + 0x08BF: "integral", + 0x08C0: "therefore", + 0x08C1: "variation", + 0x08C2: "infinity", + 0x08C5: "nabla", + 0x08C8: "approximate", + 0x08C9: "similarequal", + 0x08CD: "ifonlyif", + 0x08CE: "implies", + 0x08CF: "identical", + 0x08D6: "radical", + 0x08DA: "includedin", + 0x08DB: "includes", + 0x08DC: "intersection", + 0x08DD: "union", + 0x08DE: "logicaland", + 0x08DF: "logicalor", + 0x08EF: "partialderivative", + 0x08F6: "function", + 0x08FB: "leftarrow", + 0x08FC: "uparrow", + 0x08FD: "rightarrow", + 0x08FE: "downarrow", + 0x09DF: "blank", + 0x09E0: "soliddiamond", + 0x09E1: "checkerboard", + 0x09E2: "ht", + 0x09E3: "ff", + 0x09E4: "cr", + 0x09E5: "lf", + 0x09E8: "nl", + 0x09E9: "vt", + 0x09EA: "lowrightcorner", + 0x09EB: "uprightcorner", + 0x09EC: "upleftcorner", + 0x09ED: "lowleftcorner", + 0x09EE: "crossinglines", + 0x09EF: "horizlinescan1", + 0x09F0: "horizlinescan3", + 0x09F1: "horizlinescan5", + 0x09F2: "horizlinescan7", + 0x09F3: "horizlinescan9", + 0x09F4: "leftt", + 0x09F5: "rightt", + 0x09F6: "bott", + 0x09F7: "topt", + 0x09F8: "vertbar", + 0x0AA1: "emspace", + 0x0AA2: "enspace", + 0x0AA3: "em3space", + 0x0AA4: "em4space", + 0x0AA5: "digitspace", + 0x0AA6: "punctspace", + 0x0AA7: "thinspace", + 0x0AA8: "hairspace", + 0x0AA9: "emdash", + 0x0AAA: "endash", + 0x0AAC: "signifblank", + 0x0AAE: "ellipsis", + 0x0AAF: "doubbaselinedot", + 0x0AB0: "onethird", + 0x0AB1: "twothirds", + 0x0AB2: "onefifth", + 0x0AB3: "twofifths", + 0x0AB4: "threefifths", + 0x0AB5: "fourfifths", + 0x0AB6: "onesixth", + 0x0AB7: "fivesixths", + 0x0AB8: "careof", + 0x0ABB: "figdash", + 0x0ABC: "leftanglebracket", + 0x0ABD: "decimalpoint", + 0x0ABE: "rightanglebracket", + 0x0ABF: "marker", + 0x0AC3: "oneeighth", + 0x0AC4: "threeeighths", + 0x0AC5: "fiveeighths", + 0x0AC6: "seveneighths", + 0x0AC9: "trademark", + 0x0ACA: "signaturemark", + 0x0ACB: "trademarkincircle", + 0x0ACC: "leftopentriangle", + 0x0ACD: "rightopentriangle", + 0x0ACE: "emopencircle", + 0x0ACF: "emopenrectangle", + 0x0AD0: "leftsinglequotemark", + 0x0AD1: "rightsinglequotemark", + 0x0AD2: "leftdoublequotemark", + 0x0AD3: "rightdoublequotemark", + 0x0AD4: "prescription", + 0x0AD5: "permille", + 0x0AD6: "minutes", + 0x0AD7: "seconds", + 0x0AD9: "latincross", + 0x0ADA: "hexagram", + 0x0ADB: "filledrectbullet", + 0x0ADC: "filledlefttribullet", + 0x0ADD: "filledrighttribullet", + 0x0ADE: "emfilledcircle", + 0x0ADF: "emfilledrect", + 0x0AE0: "enopencircbullet", + 0x0AE1: "enopensquarebullet", + 0x0AE2: "openrectbullet", + 0x0AE3: "opentribulletup", + 0x0AE4: "opentribulletdown", + 0x0AE5: "openstar", + 0x0AE6: "enfilledcircbullet", + 0x0AE7: "enfilledsqbullet", + 0x0AE8: "filledtribulletup", + 0x0AE9: "filledtribulletdown", + 0x0AEA: "leftpointer", + 0x0AEB: "rightpointer", + 0x0AEC: "club", + 0x0AED: "diamond", + 0x0AEE: "heart", + 0x0AF0: "maltesecross", + 0x0AF1: "dagger", + 0x0AF2: "doubledagger", + 0x0AF3: "checkmark", + 0x0AF4: "ballotcross", + 0x0AF5: "musicalsharp", + 0x0AF6: "musicalflat", + 0x0AF7: "malesymbol", + 0x0AF8: "femalesymbol", + 0x0AF9: "telephone", + 0x0AFA: "telephonerecorder", + 0x0AFB: "phonographcopyright", + 0x0AFC: "caret", + 0x0AFD: "singlelowquotemark", + 0x0AFE: "doublelowquotemark", + 0x0AFF: "cursor", + 0x0BA3: "leftcaret", + 0x0BA6: "rightcaret", + 0x0BA8: "downcaret", + 0x0BA9: "upcaret", + 0x0BC0: "overbar", + 0x0BC2: "downtack", + 0x0BC3: "upshoe", + 0x0BC4: "downstile", + 0x0BC6: "underbar", + 0x0BCA: "jot", + 0x0BCC: "quad", + 0x0BCE: "uptack", + 0x0BCF: "circle", + 0x0BD3: "upstile", + 0x0BD6: "downshoe", + 0x0BD8: "rightshoe", + 0x0BDA: "leftshoe", + 0x0BDC: "lefttack", + 0x0BFC: "righttack", + 0x0CDF: "hebrew_doublelowline", + 0x0CE0: "hebrew_aleph", + 0x0CE1: "hebrew_bet", + 0x0CE1: "hebrew_beth", + 0x0CE2: "hebrew_gimel", + 0x0CE2: "hebrew_gimmel", + 0x0CE3: "hebrew_dalet", + 0x0CE3: "hebrew_daleth", + 0x0CE4: "hebrew_he", + 0x0CE5: "hebrew_waw", + 0x0CE6: "hebrew_zain", + 0x0CE6: "hebrew_zayin", + 0x0CE7: "hebrew_chet", + 0x0CE7: "hebrew_het", + 0x0CE8: "hebrew_tet", + 0x0CE8: "hebrew_teth", + 0x0CE9: "hebrew_yod", + 0x0CEA: "hebrew_finalkaph", + 0x0CEB: "hebrew_kaph", + 0x0CEC: "hebrew_lamed", + 0x0CED: "hebrew_finalmem", + 0x0CEE: "hebrew_mem", + 0x0CEF: "hebrew_finalnun", + 0x0CF0: "hebrew_nun", + 0x0CF1: "hebrew_samech", + 0x0CF1: "hebrew_samekh", + 0x0CF2: "hebrew_ayin", + 0x0CF3: "hebrew_finalpe", + 0x0CF4: "hebrew_pe", + 0x0CF5: "hebrew_finalzade", + 0x0CF5: "hebrew_finalzadi", + 0x0CF6: "hebrew_zade", + 0x0CF6: "hebrew_zadi", + 0x0CF7: "hebrew_qoph", + 0x0CF7: "hebrew_kuf", + 0x0CF8: "hebrew_resh", + 0x0CF9: "hebrew_shin", + 0x0CFA: "hebrew_taw", + 0x0CFA: "hebrew_taf", + 0xFF7E: "Hebrew_switch", + 0x0DA1: "Thai_kokai", + 0x0DA2: "Thai_khokhai", + 0x0DA3: "Thai_khokhuat", + 0x0DA4: "Thai_khokhwai", + 0x0DA5: "Thai_khokhon", + 0x0DA6: "Thai_khorakhang", + 0x0DA7: "Thai_ngongu", + 0x0DA8: "Thai_chochan", + 0x0DA9: "Thai_choching", + 0x0DAA: "Thai_chochang", + 0x0DAB: "Thai_soso", + 0x0DAC: "Thai_chochoe", + 0x0DAD: "Thai_yoying", + 0x0DAE: "Thai_dochada", + 0x0DAF: "Thai_topatak", + 0x0DB0: "Thai_thothan", + 0x0DB1: "Thai_thonangmontho", + 0x0DB2: "Thai_thophuthao", + 0x0DB3: "Thai_nonen", + 0x0DB4: "Thai_dodek", + 0x0DB5: "Thai_totao", + 0x0DB6: "Thai_thothung", + 0x0DB7: "Thai_thothahan", + 0x0DB8: "Thai_thothong", + 0x0DB9: "Thai_nonu", + 0x0DBA: "Thai_bobaimai", + 0x0DBB: "Thai_popla", + 0x0DBC: "Thai_phophung", + 0x0DBD: "Thai_fofa", + 0x0DBE: "Thai_phophan", + 0x0DBF: "Thai_fofan", + 0x0DC0: "Thai_phosamphao", + 0x0DC1: "Thai_moma", + 0x0DC2: "Thai_yoyak", + 0x0DC3: "Thai_rorua", + 0x0DC4: "Thai_ru", + 0x0DC5: "Thai_loling", + 0x0DC6: "Thai_lu", + 0x0DC7: "Thai_wowaen", + 0x0DC8: "Thai_sosala", + 0x0DC9: "Thai_sorusi", + 0x0DCA: "Thai_sosua", + 0x0DCB: "Thai_hohip", + 0x0DCC: "Thai_lochula", + 0x0DCD: "Thai_oang", + 0x0DCE: "Thai_honokhuk", + 0x0DCF: "Thai_paiyannoi", + 0x0DD0: "Thai_saraa", + 0x0DD1: "Thai_maihanakat", + 0x0DD2: "Thai_saraaa", + 0x0DD3: "Thai_saraam", + 0x0DD4: "Thai_sarai", + 0x0DD5: "Thai_saraii", + 0x0DD6: "Thai_saraue", + 0x0DD7: "Thai_sarauee", + 0x0DD8: "Thai_sarau", + 0x0DD9: "Thai_sarauu", + 0x0DDA: "Thai_phinthu", + 0x0DDE: "Thai_maihanakat_maitho", + 0x0DDF: "Thai_baht", + 0x0DE0: "Thai_sarae", + 0x0DE1: "Thai_saraae", + 0x0DE2: "Thai_sarao", + 0x0DE3: "Thai_saraaimaimuan", + 0x0DE4: "Thai_saraaimaimalai", + 0x0DE5: "Thai_lakkhangyao", + 0x0DE6: "Thai_maiyamok", + 0x0DE7: "Thai_maitaikhu", + 0x0DE8: "Thai_maiek", + 0x0DE9: "Thai_maitho", + 0x0DEA: "Thai_maitri", + 0x0DEB: "Thai_maichattawa", + 0x0DEC: "Thai_thanthakhat", + 0x0DED: "Thai_nikhahit", + 0x0DF0: "Thai_leksun", + 0x0DF1: "Thai_leknung", + 0x0DF2: "Thai_leksong", + 0x0DF3: "Thai_leksam", + 0x0DF4: "Thai_leksi", + 0x0DF5: "Thai_lekha", + 0x0DF6: "Thai_lekhok", + 0x0DF7: "Thai_lekchet", + 0x0DF8: "Thai_lekpaet", + 0x0DF9: "Thai_lekkao", + 0xFF31: "Hangul", + 0xFF32: "Hangul_Start", + 0xFF33: "Hangul_End", + 0xFF34: "Hangul_Hanja", + 0xFF35: "Hangul_Jamo", + 0xFF36: "Hangul_Romaja", + 0xFF37: "Hangul_Codeinput", + 0xFF38: "Hangul_Jeonja", + 0xFF39: "Hangul_Banja", + 0xFF3A: "Hangul_PreHanja", + 0xFF3B: "Hangul_PostHanja", + 0xFF3C: "Hangul_SingleCandidate", + 0xFF3D: "Hangul_MultipleCandidate", + 0xFF3E: "Hangul_PreviousCandidate", + 0xFF3F: "Hangul_Special", + 0xFF7E: "Hangul_switch", + 0x0EA1: "Hangul_Kiyeog", + 0x0EA2: "Hangul_SsangKiyeog", + 0x0EA3: "Hangul_KiyeogSios", + 0x0EA4: "Hangul_Nieun", + 0x0EA5: "Hangul_NieunJieuj", + 0x0EA6: "Hangul_NieunHieuh", + 0x0EA7: "Hangul_Dikeud", + 0x0EA8: "Hangul_SsangDikeud", + 0x0EA9: "Hangul_Rieul", + 0x0EAA: "Hangul_RieulKiyeog", + 0x0EAB: "Hangul_RieulMieum", + 0x0EAC: "Hangul_RieulPieub", + 0x0EAD: "Hangul_RieulSios", + 0x0EAE: "Hangul_RieulTieut", + 0x0EAF: "Hangul_RieulPhieuf", + 0x0EB0: "Hangul_RieulHieuh", + 0x0EB1: "Hangul_Mieum", + 0x0EB2: "Hangul_Pieub", + 0x0EB3: "Hangul_SsangPieub", + 0x0EB4: "Hangul_PieubSios", + 0x0EB5: "Hangul_Sios", + 0x0EB6: "Hangul_SsangSios", + 0x0EB7: "Hangul_Ieung", + 0x0EB8: "Hangul_Jieuj", + 0x0EB9: "Hangul_SsangJieuj", + 0x0EBA: "Hangul_Cieuc", + 0x0EBB: "Hangul_Khieuq", + 0x0EBC: "Hangul_Tieut", + 0x0EBD: "Hangul_Phieuf", + 0x0EBE: "Hangul_Hieuh", + 0x0EBF: "Hangul_A", + 0x0EC0: "Hangul_AE", + 0x0EC1: "Hangul_YA", + 0x0EC2: "Hangul_YAE", + 0x0EC3: "Hangul_EO", + 0x0EC4: "Hangul_E", + 0x0EC5: "Hangul_YEO", + 0x0EC6: "Hangul_YE", + 0x0EC7: "Hangul_O", + 0x0EC8: "Hangul_WA", + 0x0EC9: "Hangul_WAE", + 0x0ECA: "Hangul_OE", + 0x0ECB: "Hangul_YO", + 0x0ECC: "Hangul_U", + 0x0ECD: "Hangul_WEO", + 0x0ECE: "Hangul_WE", + 0x0ECF: "Hangul_WI", + 0x0ED0: "Hangul_YU", + 0x0ED1: "Hangul_EU", + 0x0ED2: "Hangul_YI", + 0x0ED3: "Hangul_I", + 0x0ED4: "Hangul_J_Kiyeog", + 0x0ED5: "Hangul_J_SsangKiyeog", + 0x0ED6: "Hangul_J_KiyeogSios", + 0x0ED7: "Hangul_J_Nieun", + 0x0ED8: "Hangul_J_NieunJieuj", + 0x0ED9: "Hangul_J_NieunHieuh", + 0x0EDA: "Hangul_J_Dikeud", + 0x0EDB: "Hangul_J_Rieul", + 0x0EDC: "Hangul_J_RieulKiyeog", + 0x0EDD: "Hangul_J_RieulMieum", + 0x0EDE: "Hangul_J_RieulPieub", + 0x0EDF: "Hangul_J_RieulSios", + 0x0EE0: "Hangul_J_RieulTieut", + 0x0EE1: "Hangul_J_RieulPhieuf", + 0x0EE2: "Hangul_J_RieulHieuh", + 0x0EE3: "Hangul_J_Mieum", + 0x0EE4: "Hangul_J_Pieub", + 0x0EE5: "Hangul_J_PieubSios", + 0x0EE6: "Hangul_J_Sios", + 0x0EE7: "Hangul_J_SsangSios", + 0x0EE8: "Hangul_J_Ieung", + 0x0EE9: "Hangul_J_Jieuj", + 0x0EEA: "Hangul_J_Cieuc", + 0x0EEB: "Hangul_J_Khieuq", + 0x0EEC: "Hangul_J_Tieut", + 0x0EED: "Hangul_J_Phieuf", + 0x0EEE: "Hangul_J_Hieuh", + 0x0EEF: "Hangul_RieulYeorinHieuh", + 0x0EF0: "Hangul_SunkyeongeumMieum", + 0x0EF1: "Hangul_SunkyeongeumPieub", + 0x0EF2: "Hangul_PanSios", + 0x0EF3: "Hangul_KkogjiDalrinIeung", + 0x0EF4: "Hangul_SunkyeongeumPhieuf", + 0x0EF5: "Hangul_YeorinHieuh", + 0x0EF6: "Hangul_AraeA", + 0x0EF7: "Hangul_AraeAE", + 0x0EF8: "Hangul_J_PanSios", + 0x0EF9: "Hangul_J_KkogjiDalrinIeung", + 0x0EFA: "Hangul_J_YeorinHieuh", + 0x0EFF: "Korean_Won", + 0x1000587: "Armenian_ligature_ew", + 0x1000589: "Armenian_full_stop", + 0x1000589: "Armenian_verjaket", + 0x100055D: "Armenian_separation_mark", + 0x100055D: "Armenian_but", + 0x100058A: "Armenian_hyphen", + 0x100058A: "Armenian_yentamna", + 0x100055C: "Armenian_exclam", + 0x100055C: "Armenian_amanak", + 0x100055B: "Armenian_accent", + 0x100055B: "Armenian_shesht", + 0x100055E: "Armenian_question", + 0x100055E: "Armenian_paruyk", + 0x1000531: "Armenian_AYB", + 0x1000561: "Armenian_ayb", + 0x1000532: "Armenian_BEN", + 0x1000562: "Armenian_ben", + 0x1000533: "Armenian_GIM", + 0x1000563: "Armenian_gim", + 0x1000534: "Armenian_DA", + 0x1000564: "Armenian_da", + 0x1000535: "Armenian_YECH", + 0x1000565: "Armenian_yech", + 0x1000536: "Armenian_ZA", + 0x1000566: "Armenian_za", + 0x1000537: "Armenian_E", + 0x1000567: "Armenian_e", + 0x1000538: "Armenian_AT", + 0x1000568: "Armenian_at", + 0x1000539: "Armenian_TO", + 0x1000569: "Armenian_to", + 0x100053A: "Armenian_ZHE", + 0x100056A: "Armenian_zhe", + 0x100053B: "Armenian_INI", + 0x100056B: "Armenian_ini", + 0x100053C: "Armenian_LYUN", + 0x100056C: "Armenian_lyun", + 0x100053D: "Armenian_KHE", + 0x100056D: "Armenian_khe", + 0x100053E: "Armenian_TSA", + 0x100056E: "Armenian_tsa", + 0x100053F: "Armenian_KEN", + 0x100056F: "Armenian_ken", + 0x1000540: "Armenian_HO", + 0x1000570: "Armenian_ho", + 0x1000541: "Armenian_DZA", + 0x1000571: "Armenian_dza", + 0x1000542: "Armenian_GHAT", + 0x1000572: "Armenian_ghat", + 0x1000543: "Armenian_TCHE", + 0x1000573: "Armenian_tche", + 0x1000544: "Armenian_MEN", + 0x1000574: "Armenian_men", + 0x1000545: "Armenian_HI", + 0x1000575: "Armenian_hi", + 0x1000546: "Armenian_NU", + 0x1000576: "Armenian_nu", + 0x1000547: "Armenian_SHA", + 0x1000577: "Armenian_sha", + 0x1000548: "Armenian_VO", + 0x1000578: "Armenian_vo", + 0x1000549: "Armenian_CHA", + 0x1000579: "Armenian_cha", + 0x100054A: "Armenian_PE", + 0x100057A: "Armenian_pe", + 0x100054B: "Armenian_JE", + 0x100057B: "Armenian_je", + 0x100054C: "Armenian_RA", + 0x100057C: "Armenian_ra", + 0x100054D: "Armenian_SE", + 0x100057D: "Armenian_se", + 0x100054E: "Armenian_VEV", + 0x100057E: "Armenian_vev", + 0x100054F: "Armenian_TYUN", + 0x100057F: "Armenian_tyun", + 0x1000550: "Armenian_RE", + 0x1000580: "Armenian_re", + 0x1000551: "Armenian_TSO", + 0x1000581: "Armenian_tso", + 0x1000552: "Armenian_VYUN", + 0x1000582: "Armenian_vyun", + 0x1000553: "Armenian_PYUR", + 0x1000583: "Armenian_pyur", + 0x1000554: "Armenian_KE", + 0x1000584: "Armenian_ke", + 0x1000555: "Armenian_O", + 0x1000585: "Armenian_o", + 0x1000556: "Armenian_FE", + 0x1000586: "Armenian_fe", + 0x100055A: "Armenian_apostrophe", + 0x10010D0: "Georgian_an", + 0x10010D1: "Georgian_ban", + 0x10010D2: "Georgian_gan", + 0x10010D3: "Georgian_don", + 0x10010D4: "Georgian_en", + 0x10010D5: "Georgian_vin", + 0x10010D6: "Georgian_zen", + 0x10010D7: "Georgian_tan", + 0x10010D8: "Georgian_in", + 0x10010D9: "Georgian_kan", + 0x10010DA: "Georgian_las", + 0x10010DB: "Georgian_man", + 0x10010DC: "Georgian_nar", + 0x10010DD: "Georgian_on", + 0x10010DE: "Georgian_par", + 0x10010DF: "Georgian_zhar", + 0x10010E0: "Georgian_rae", + 0x10010E1: "Georgian_san", + 0x10010E2: "Georgian_tar", + 0x10010E3: "Georgian_un", + 0x10010E4: "Georgian_phar", + 0x10010E5: "Georgian_khar", + 0x10010E6: "Georgian_ghan", + 0x10010E7: "Georgian_qar", + 0x10010E8: "Georgian_shin", + 0x10010E9: "Georgian_chin", + 0x10010EA: "Georgian_can", + 0x10010EB: "Georgian_jil", + 0x10010EC: "Georgian_cil", + 0x10010ED: "Georgian_char", + 0x10010EE: "Georgian_xan", + 0x10010EF: "Georgian_jhan", + 0x10010F0: "Georgian_hae", + 0x10010F1: "Georgian_he", + 0x10010F2: "Georgian_hie", + 0x10010F3: "Georgian_we", + 0x10010F4: "Georgian_har", + 0x10010F5: "Georgian_hoe", + 0x10010F6: "Georgian_fi", + 0x1001E8A: "Xabovedot", + 0x100012C: "Ibreve", + 0x10001B5: "Zstroke", + 0x10001E6: "Gcaron", + 0x10001D1: "Ocaron", + 0x100019F: "Obarred", + 0x1001E8B: "xabovedot", + 0x100012D: "ibreve", + 0x10001B6: "zstroke", + 0x10001E7: "gcaron", + 0x10001D2: "ocaron", + 0x1000275: "obarred", + 0x100018F: "SCHWA", + 0x1000259: "schwa", + 0x10001B7: "EZH", + 0x1000292: "ezh", + 0x1001E36: "Lbelowdot", + 0x1001E37: "lbelowdot", + 0x1001EA0: "Abelowdot", + 0x1001EA1: "abelowdot", + 0x1001EA2: "Ahook", + 0x1001EA3: "ahook", + 0x1001EA4: "Acircumflexacute", + 0x1001EA5: "acircumflexacute", + 0x1001EA6: "Acircumflexgrave", + 0x1001EA7: "acircumflexgrave", + 0x1001EA8: "Acircumflexhook", + 0x1001EA9: "acircumflexhook", + 0x1001EAA: "Acircumflextilde", + 0x1001EAB: "acircumflextilde", + 0x1001EAC: "Acircumflexbelowdot", + 0x1001EAD: "acircumflexbelowdot", + 0x1001EAE: "Abreveacute", + 0x1001EAF: "abreveacute", + 0x1001EB0: "Abrevegrave", + 0x1001EB1: "abrevegrave", + 0x1001EB2: "Abrevehook", + 0x1001EB3: "abrevehook", + 0x1001EB4: "Abrevetilde", + 0x1001EB5: "abrevetilde", + 0x1001EB6: "Abrevebelowdot", + 0x1001EB7: "abrevebelowdot", + 0x1001EB8: "Ebelowdot", + 0x1001EB9: "ebelowdot", + 0x1001EBA: "Ehook", + 0x1001EBB: "ehook", + 0x1001EBC: "Etilde", + 0x1001EBD: "etilde", + 0x1001EBE: "Ecircumflexacute", + 0x1001EBF: "ecircumflexacute", + 0x1001EC0: "Ecircumflexgrave", + 0x1001EC1: "ecircumflexgrave", + 0x1001EC2: "Ecircumflexhook", + 0x1001EC3: "ecircumflexhook", + 0x1001EC4: "Ecircumflextilde", + 0x1001EC5: "ecircumflextilde", + 0x1001EC6: "Ecircumflexbelowdot", + 0x1001EC7: "ecircumflexbelowdot", + 0x1001EC8: "Ihook", + 0x1001EC9: "ihook", + 0x1001ECA: "Ibelowdot", + 0x1001ECB: "ibelowdot", + 0x1001ECC: "Obelowdot", + 0x1001ECD: "obelowdot", + 0x1001ECE: "Ohook", + 0x1001ECF: "ohook", + 0x1001ED0: "Ocircumflexacute", + 0x1001ED1: "ocircumflexacute", + 0x1001ED2: "Ocircumflexgrave", + 0x1001ED3: "ocircumflexgrave", + 0x1001ED4: "Ocircumflexhook", + 0x1001ED5: "ocircumflexhook", + 0x1001ED6: "Ocircumflextilde", + 0x1001ED7: "ocircumflextilde", + 0x1001ED8: "Ocircumflexbelowdot", + 0x1001ED9: "ocircumflexbelowdot", + 0x1001EDA: "Ohornacute", + 0x1001EDB: "ohornacute", + 0x1001EDC: "Ohorngrave", + 0x1001EDD: "ohorngrave", + 0x1001EDE: "Ohornhook", + 0x1001EDF: "ohornhook", + 0x1001EE0: "Ohorntilde", + 0x1001EE1: "ohorntilde", + 0x1001EE2: "Ohornbelowdot", + 0x1001EE3: "ohornbelowdot", + 0x1001EE4: "Ubelowdot", + 0x1001EE5: "ubelowdot", + 0x1001EE6: "Uhook", + 0x1001EE7: "uhook", + 0x1001EE8: "Uhornacute", + 0x1001EE9: "uhornacute", + 0x1001EEA: "Uhorngrave", + 0x1001EEB: "uhorngrave", + 0x1001EEC: "Uhornhook", + 0x1001EED: "uhornhook", + 0x1001EEE: "Uhorntilde", + 0x1001EEF: "uhorntilde", + 0x1001EF0: "Uhornbelowdot", + 0x1001EF1: "uhornbelowdot", + 0x1001EF4: "Ybelowdot", + 0x1001EF5: "ybelowdot", + 0x1001EF6: "Yhook", + 0x1001EF7: "yhook", + 0x1001EF8: "Ytilde", + 0x1001EF9: "ytilde", + 0x10001A0: "Ohorn", + 0x10001A1: "ohorn", + 0x10001AF: "Uhorn", + 0x10001B0: "uhorn", + 0x10020A0: "EcuSign", + 0x10020A1: "ColonSign", + 0x10020A2: "CruzeiroSign", + 0x10020A3: "FFrancSign", + 0x10020A4: "LiraSign", + 0x10020A5: "MillSign", + 0x10020A6: "NairaSign", + 0x10020A7: "PesetaSign", + 0x10020A8: "RupeeSign", + 0x10020A9: "WonSign", + 0x10020AA: "NewSheqelSign", + 0x10020AB: "DongSign", + 0x20AC: "EuroSign", + 0x1002070: "zerosuperior", + 0x1002074: "foursuperior", + 0x1002075: "fivesuperior", + 0x1002076: "sixsuperior", + 0x1002077: "sevensuperior", + 0x1002078: "eightsuperior", + 0x1002079: "ninesuperior", + 0x1002080: "zerosubscript", + 0x1002081: "onesubscript", + 0x1002082: "twosubscript", + 0x1002083: "threesubscript", + 0x1002084: "foursubscript", + 0x1002085: "fivesubscript", + 0x1002086: "sixsubscript", + 0x1002087: "sevensubscript", + 0x1002088: "eightsubscript", + 0x1002089: "ninesubscript", + 0x1002202: "partdifferential", + 0x1002205: "emptyset", + 0x1002208: "elementof", + 0x1002209: "notelementof", + 0x100220B: "containsas", + 0x100221A: "squareroot", + 0x100221B: "cuberoot", + 0x100221C: "fourthroot", + 0x100222C: "dintegral", + 0x100222D: "tintegral", + 0x1002235: "because", + 0x1002248: "approxeq", + 0x1002247: "notapproxeq", + 0x1002262: "notidentical", + 0x1002263: "stricteq", + 0xFFF1: "braille_dot_1", + 0xFFF2: "braille_dot_2", + 0xFFF3: "braille_dot_3", + 0xFFF4: "braille_dot_4", + 0xFFF5: "braille_dot_5", + 0xFFF6: "braille_dot_6", + 0xFFF7: "braille_dot_7", + 0xFFF8: "braille_dot_8", + 0xFFF9: "braille_dot_9", + 0xFFFA: "braille_dot_10", + 0x1002800: "braille_blank", + 0x1002801: "braille_dots_1", + 0x1002802: "braille_dots_2", + 0x1002803: "braille_dots_12", + 0x1002804: "braille_dots_3", + 0x1002805: "braille_dots_13", + 0x1002806: "braille_dots_23", + 0x1002807: "braille_dots_123", + 0x1002808: "braille_dots_4", + 0x1002809: "braille_dots_14", + 0x100280A: "braille_dots_24", + 0x100280B: "braille_dots_124", + 0x100280C: "braille_dots_34", + 0x100280D: "braille_dots_134", + 0x100280E: "braille_dots_234", + 0x100280F: "braille_dots_1234", + 0x1002810: "braille_dots_5", + 0x1002811: "braille_dots_15", + 0x1002812: "braille_dots_25", + 0x1002813: "braille_dots_125", + 0x1002814: "braille_dots_35", + 0x1002815: "braille_dots_135", + 0x1002816: "braille_dots_235", + 0x1002817: "braille_dots_1235", + 0x1002818: "braille_dots_45", + 0x1002819: "braille_dots_145", + 0x100281A: "braille_dots_245", + 0x100281B: "braille_dots_1245", + 0x100281C: "braille_dots_345", + 0x100281D: "braille_dots_1345", + 0x100281E: "braille_dots_2345", + 0x100281F: "braille_dots_12345", + 0x1002820: "braille_dots_6", + 0x1002821: "braille_dots_16", + 0x1002822: "braille_dots_26", + 0x1002823: "braille_dots_126", + 0x1002824: "braille_dots_36", + 0x1002825: "braille_dots_136", + 0x1002826: "braille_dots_236", + 0x1002827: "braille_dots_1236", + 0x1002828: "braille_dots_46", + 0x1002829: "braille_dots_146", + 0x100282A: "braille_dots_246", + 0x100282B: "braille_dots_1246", + 0x100282C: "braille_dots_346", + 0x100282D: "braille_dots_1346", + 0x100282E: "braille_dots_2346", + 0x100282F: "braille_dots_12346", + 0x1002830: "braille_dots_56", + 0x1002831: "braille_dots_156", + 0x1002832: "braille_dots_256", + 0x1002833: "braille_dots_1256", + 0x1002834: "braille_dots_356", + 0x1002835: "braille_dots_1356", + 0x1002836: "braille_dots_2356", + 0x1002837: "braille_dots_12356", + 0x1002838: "braille_dots_456", + 0x1002839: "braille_dots_1456", + 0x100283A: "braille_dots_2456", + 0x100283B: "braille_dots_12456", + 0x100283C: "braille_dots_3456", + 0x100283D: "braille_dots_13456", + 0x100283E: "braille_dots_23456", + 0x100283F: "braille_dots_123456", + 0x1002840: "braille_dots_7", + 0x1002841: "braille_dots_17", + 0x1002842: "braille_dots_27", + 0x1002843: "braille_dots_127", + 0x1002844: "braille_dots_37", + 0x1002845: "braille_dots_137", + 0x1002846: "braille_dots_237", + 0x1002847: "braille_dots_1237", + 0x1002848: "braille_dots_47", + 0x1002849: "braille_dots_147", + 0x100284A: "braille_dots_247", + 0x100284B: "braille_dots_1247", + 0x100284C: "braille_dots_347", + 0x100284D: "braille_dots_1347", + 0x100284E: "braille_dots_2347", + 0x100284F: "braille_dots_12347", + 0x1002850: "braille_dots_57", + 0x1002851: "braille_dots_157", + 0x1002852: "braille_dots_257", + 0x1002853: "braille_dots_1257", + 0x1002854: "braille_dots_357", + 0x1002855: "braille_dots_1357", + 0x1002856: "braille_dots_2357", + 0x1002857: "braille_dots_12357", + 0x1002858: "braille_dots_457", + 0x1002859: "braille_dots_1457", + 0x100285A: "braille_dots_2457", + 0x100285B: "braille_dots_12457", + 0x100285C: "braille_dots_3457", + 0x100285D: "braille_dots_13457", + 0x100285E: "braille_dots_23457", + 0x100285F: "braille_dots_123457", + 0x1002860: "braille_dots_67", + 0x1002861: "braille_dots_167", + 0x1002862: "braille_dots_267", + 0x1002863: "braille_dots_1267", + 0x1002864: "braille_dots_367", + 0x1002865: "braille_dots_1367", + 0x1002866: "braille_dots_2367", + 0x1002867: "braille_dots_12367", + 0x1002868: "braille_dots_467", + 0x1002869: "braille_dots_1467", + 0x100286A: "braille_dots_2467", + 0x100286B: "braille_dots_12467", + 0x100286C: "braille_dots_3467", + 0x100286D: "braille_dots_13467", + 0x100286E: "braille_dots_23467", + 0x100286F: "braille_dots_123467", + 0x1002870: "braille_dots_567", + 0x1002871: "braille_dots_1567", + 0x1002872: "braille_dots_2567", + 0x1002873: "braille_dots_12567", + 0x1002874: "braille_dots_3567", + 0x1002875: "braille_dots_13567", + 0x1002876: "braille_dots_23567", + 0x1002877: "braille_dots_123567", + 0x1002878: "braille_dots_4567", + 0x1002879: "braille_dots_14567", + 0x100287A: "braille_dots_24567", + 0x100287B: "braille_dots_124567", + 0x100287C: "braille_dots_34567", + 0x100287D: "braille_dots_134567", + 0x100287E: "braille_dots_234567", + 0x100287F: "braille_dots_1234567", + 0x1002880: "braille_dots_8", + 0x1002881: "braille_dots_18", + 0x1002882: "braille_dots_28", + 0x1002883: "braille_dots_128", + 0x1002884: "braille_dots_38", + 0x1002885: "braille_dots_138", + 0x1002886: "braille_dots_238", + 0x1002887: "braille_dots_1238", + 0x1002888: "braille_dots_48", + 0x1002889: "braille_dots_148", + 0x100288A: "braille_dots_248", + 0x100288B: "braille_dots_1248", + 0x100288C: "braille_dots_348", + 0x100288D: "braille_dots_1348", + 0x100288E: "braille_dots_2348", + 0x100288F: "braille_dots_12348", + 0x1002890: "braille_dots_58", + 0x1002891: "braille_dots_158", + 0x1002892: "braille_dots_258", + 0x1002893: "braille_dots_1258", + 0x1002894: "braille_dots_358", + 0x1002895: "braille_dots_1358", + 0x1002896: "braille_dots_2358", + 0x1002897: "braille_dots_12358", + 0x1002898: "braille_dots_458", + 0x1002899: "braille_dots_1458", + 0x100289A: "braille_dots_2458", + 0x100289B: "braille_dots_12458", + 0x100289C: "braille_dots_3458", + 0x100289D: "braille_dots_13458", + 0x100289E: "braille_dots_23458", + 0x100289F: "braille_dots_123458", + 0x10028A0: "braille_dots_68", + 0x10028A1: "braille_dots_168", + 0x10028A2: "braille_dots_268", + 0x10028A3: "braille_dots_1268", + 0x10028A4: "braille_dots_368", + 0x10028A5: "braille_dots_1368", + 0x10028A6: "braille_dots_2368", + 0x10028A7: "braille_dots_12368", + 0x10028A8: "braille_dots_468", + 0x10028A9: "braille_dots_1468", + 0x10028AA: "braille_dots_2468", + 0x10028AB: "braille_dots_12468", + 0x10028AC: "braille_dots_3468", + 0x10028AD: "braille_dots_13468", + 0x10028AE: "braille_dots_23468", + 0x10028AF: "braille_dots_123468", + 0x10028B0: "braille_dots_568", + 0x10028B1: "braille_dots_1568", + 0x10028B2: "braille_dots_2568", + 0x10028B3: "braille_dots_12568", + 0x10028B4: "braille_dots_3568", + 0x10028B5: "braille_dots_13568", + 0x10028B6: "braille_dots_23568", + 0x10028B7: "braille_dots_123568", + 0x10028B8: "braille_dots_4568", + 0x10028B9: "braille_dots_14568", + 0x10028BA: "braille_dots_24568", + 0x10028BB: "braille_dots_124568", + 0x10028BC: "braille_dots_34568", + 0x10028BD: "braille_dots_134568", + 0x10028BE: "braille_dots_234568", + 0x10028BF: "braille_dots_1234568", + 0x10028C0: "braille_dots_78", + 0x10028C1: "braille_dots_178", + 0x10028C2: "braille_dots_278", + 0x10028C3: "braille_dots_1278", + 0x10028C4: "braille_dots_378", + 0x10028C5: "braille_dots_1378", + 0x10028C6: "braille_dots_2378", + 0x10028C7: "braille_dots_12378", + 0x10028C8: "braille_dots_478", + 0x10028C9: "braille_dots_1478", + 0x10028CA: "braille_dots_2478", + 0x10028CB: "braille_dots_12478", + 0x10028CC: "braille_dots_3478", + 0x10028CD: "braille_dots_13478", + 0x10028CE: "braille_dots_23478", + 0x10028CF: "braille_dots_123478", + 0x10028D0: "braille_dots_578", + 0x10028D1: "braille_dots_1578", + 0x10028D2: "braille_dots_2578", + 0x10028D3: "braille_dots_12578", + 0x10028D4: "braille_dots_3578", + 0x10028D5: "braille_dots_13578", + 0x10028D6: "braille_dots_23578", + 0x10028D7: "braille_dots_123578", + 0x10028D8: "braille_dots_4578", + 0x10028D9: "braille_dots_14578", + 0x10028DA: "braille_dots_24578", + 0x10028DB: "braille_dots_124578", + 0x10028DC: "braille_dots_34578", + 0x10028DD: "braille_dots_134578", + 0x10028DE: "braille_dots_234578", + 0x10028DF: "braille_dots_1234578", + 0x10028E0: "braille_dots_678", + 0x10028E1: "braille_dots_1678", + 0x10028E2: "braille_dots_2678", + 0x10028E3: "braille_dots_12678", + 0x10028E4: "braille_dots_3678", + 0x10028E5: "braille_dots_13678", + 0x10028E6: "braille_dots_23678", + 0x10028E7: "braille_dots_123678", + 0x10028E8: "braille_dots_4678", + 0x10028E9: "braille_dots_14678", + 0x10028EA: "braille_dots_24678", + 0x10028EB: "braille_dots_124678", + 0x10028EC: "braille_dots_34678", + 0x10028ED: "braille_dots_134678", + 0x10028EE: "braille_dots_234678", + 0x10028EF: "braille_dots_1234678", + 0x10028F0: "braille_dots_5678", + 0x10028F1: "braille_dots_15678", + 0x10028F2: "braille_dots_25678", + 0x10028F3: "braille_dots_125678", + 0x10028F4: "braille_dots_35678", + 0x10028F5: "braille_dots_135678", + 0x10028F6: "braille_dots_235678", + 0x10028F7: "braille_dots_1235678", + 0x10028F8: "braille_dots_45678", + 0x10028F9: "braille_dots_145678", + 0x10028FA: "braille_dots_245678", + 0x10028FB: "braille_dots_1245678", + 0x10028FC: "braille_dots_345678", + 0x10028FD: "braille_dots_1345678", + 0x10028FE: "braille_dots_2345678", + 0x10028FF: "braille_dots_12345678", + 0x1000D82: "Sinh_ng", + 0x1000D83: "Sinh_h2", + 0x1000D85: "Sinh_a", + 0x1000D86: "Sinh_aa", + 0x1000D87: "Sinh_ae", + 0x1000D88: "Sinh_aee", + 0x1000D89: "Sinh_i", + 0x1000D8A: "Sinh_ii", + 0x1000D8B: "Sinh_u", + 0x1000D8C: "Sinh_uu", + 0x1000D8D: "Sinh_ri", + 0x1000D8E: "Sinh_rii", + 0x1000D8F: "Sinh_lu", + 0x1000D90: "Sinh_luu", + 0x1000D91: "Sinh_e", + 0x1000D92: "Sinh_ee", + 0x1000D93: "Sinh_ai", + 0x1000D94: "Sinh_o", + 0x1000D95: "Sinh_oo", + 0x1000D96: "Sinh_au", + 0x1000D9A: "Sinh_ka", + 0x1000D9B: "Sinh_kha", + 0x1000D9C: "Sinh_ga", + 0x1000D9D: "Sinh_gha", + 0x1000D9E: "Sinh_ng2", + 0x1000D9F: "Sinh_nga", + 0x1000DA0: "Sinh_ca", + 0x1000DA1: "Sinh_cha", + 0x1000DA2: "Sinh_ja", + 0x1000DA3: "Sinh_jha", + 0x1000DA4: "Sinh_nya", + 0x1000DA5: "Sinh_jnya", + 0x1000DA6: "Sinh_nja", + 0x1000DA7: "Sinh_tta", + 0x1000DA8: "Sinh_ttha", + 0x1000DA9: "Sinh_dda", + 0x1000DAA: "Sinh_ddha", + 0x1000DAB: "Sinh_nna", + 0x1000DAC: "Sinh_ndda", + 0x1000DAD: "Sinh_tha", + 0x1000DAE: "Sinh_thha", + 0x1000DAF: "Sinh_dha", + 0x1000DB0: "Sinh_dhha", + 0x1000DB1: "Sinh_na", + 0x1000DB3: "Sinh_ndha", + 0x1000DB4: "Sinh_pa", + 0x1000DB5: "Sinh_pha", + 0x1000DB6: "Sinh_ba", + 0x1000DB7: "Sinh_bha", + 0x1000DB8: "Sinh_ma", + 0x1000DB9: "Sinh_mba", + 0x1000DBA: "Sinh_ya", + 0x1000DBB: "Sinh_ra", + 0x1000DBD: "Sinh_la", + 0x1000DC0: "Sinh_va", + 0x1000DC1: "Sinh_sha", + 0x1000DC2: "Sinh_ssha", + 0x1000DC3: "Sinh_sa", + 0x1000DC4: "Sinh_ha", + 0x1000DC5: "Sinh_lla", + 0x1000DC6: "Sinh_fa", + 0x1000DCA: "Sinh_al", + 0x1000DCF: "Sinh_aa2", + 0x1000DD0: "Sinh_ae2", + 0x1000DD1: "Sinh_aee2", + 0x1000DD2: "Sinh_i2", + 0x1000DD3: "Sinh_ii2", + 0x1000DD4: "Sinh_u2", + 0x1000DD6: "Sinh_uu2", + 0x1000DD8: "Sinh_ru2", + 0x1000DD9: "Sinh_e2", + 0x1000DDA: "Sinh_ee2", + 0x1000DDB: "Sinh_ai2", + 0x1000DDC: "Sinh_o2", + 0x1000DDD: "Sinh_oo2", + 0x1000DDE: "Sinh_au2", + 0x1000DDF: "Sinh_lu2", + 0x1000DF2: "Sinh_ruu2", + 0x1000DF3: "Sinh_luu2", + 0x1000DF4: "Sinh_kunddaliya", + }; + + /** + * All keysyms which should not repeat when held down. + * @private + */ + var no_repeat = { + 0xFE03: true, // ISO Level 3 Shift (AltGr) + 0xFFE1: true, // Left shift + 0xFFE2: true, // Right shift + 0xFFE3: true, // Left ctrl + 0xFFE4: true, // Right ctrl + 0xFFE7: true, // Left meta + 0xFFE8: true, // Right meta + 0xFFE9: true, // Left alt + 0xFFEA: true, // Right alt + 0xFFEB: true, // Left hyper + 0xFFEC: true // Right hyper + }; + + /** + * All modifiers and their states. + */ + this.modifiers = new Keyboard.ModifierState(); + + /** + * The state of every key, indexed by keysym. If a particular key is + * pressed, the value of pressed for that keysym will be true. If a key + * is not currently pressed, it will not be defined. + */ + this.pressed = {}; + + /** + * The last result of calling the onkeydown handler for each key, indexed + * by keysym. This is used to prevent/allow default actions for key events, + * even when the onkeydown handler cannot be called again because the key + * is (theoretically) still pressed. + * + * @private + */ + var last_keydown_result = {}; + + /** + * The keysym most recently associated with a given keycode when keydown + * fired. This object maps keycodes to keysyms. + * + * @private + * @type {Object.<Number, Number>} + */ + var recentKeysym = {}; + + /** + * Timeout before key repeat starts. + * @private + */ + var key_repeat_timeout = null; + + /** + * Interval which presses and releases the last key pressed while that + * key is still being held down. + * @private + */ + var key_repeat_interval = null; + + /** + * Given an array of keysyms indexed by location, returns the keysym + * for the given location, or the keysym for the standard location if + * undefined. + * + * @private + * @param {Number[]} keysyms + * An array of keysyms, where the index of the keysym in the array is + * the location value. + * + * @param {Number} location + * The location on the keyboard corresponding to the key pressed, as + * defined at: http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + */ + var get_keysym = function get_keysym(keysyms, location) { + + if (!keysyms) + return null; + + return keysyms[location] || keysyms[0]; + }; + + /** + * Returns true if the given keysym corresponds to a printable character, + * false otherwise. + * + * @param {Number} keysym + * The keysym to check. + * + * @returns {Boolean} + * true if the given keysym corresponds to a printable character, + * false otherwise. + */ + var isPrintable = function isPrintable(keysym) { + + // Keysyms with Unicode equivalents are printable + return (keysym >= 0x00 && keysym <= 0xFF) + || (keysym & 0xFFFF0000) === 0x01000000; + + }; + + function keysym_from_key_identifier(identifier, location, shifted) { + + if (!identifier) + return null; + + var typedCharacter; + + // If identifier is U+xxxx, decode Unicode character + var unicodePrefixLocation = identifier.indexOf("U+"); + if (unicodePrefixLocation >= 0) { + var hex = identifier.substring(unicodePrefixLocation+2); + typedCharacter = String.fromCharCode(parseInt(hex, 16)); + } + + // If single character and not keypad, use that as typed character + else if (identifier.length === 1 && location !== 3) + typedCharacter = identifier; + + // Otherwise, look up corresponding keysym + else + return get_keysym(keyidentifier_keysym[identifier], location); + + // Alter case if necessary + if (shifted === true) + typedCharacter = typedCharacter.toUpperCase(); + else if (shifted === false) + typedCharacter = typedCharacter.toLowerCase(); + + // Get codepoint + var codepoint = typedCharacter.charCodeAt(0); + return keysym_from_charcode(codepoint); + + } + + function isControlCharacter(codepoint) { + return codepoint <= 0x1F || (codepoint >= 0x7F && codepoint <= 0x9F); + } + + function keysym_from_charcode(codepoint) { + + // Keysyms for control characters + if (isControlCharacter(codepoint)) return 0xFF00 | codepoint; + + // Keysyms for ASCII chars + if (codepoint >= 0x0000 && codepoint <= 0x00FF) + return codepoint; + + // Keysyms for Unicode + if (codepoint >= 0x0100 && codepoint <= 0x10FFFF) + return 0x01000000 | codepoint; + + return null; + + } + + function keysym_from_keycode(keyCode, location) { + return get_keysym(keycodeKeysyms[keyCode], location); + } + + /** + * Heuristically detects if the legacy keyIdentifier property of + * a keydown/keyup event looks incorrectly derived. Chrome, and + * presumably others, will produce the keyIdentifier by assuming + * the keyCode is the Unicode codepoint for that key. This is not + * correct in all cases. + * + * @private + * @param {Number} keyCode + * The keyCode from a browser keydown/keyup event. + * + * @param {String} keyIdentifier + * The legacy keyIdentifier from a browser keydown/keyup event. + * + * @returns {Boolean} + * true if the keyIdentifier looks sane, false if the keyIdentifier + * appears incorrectly derived or is missing entirely. + */ + var key_identifier_sane = function key_identifier_sane(keyCode, keyIdentifier) { + + // Missing identifier is not sane + if (!keyIdentifier) + return false; + + // Assume non-Unicode keyIdentifier values are sane + var unicodePrefixLocation = keyIdentifier.indexOf("U+"); + if (unicodePrefixLocation === -1) + return true; + + // If the Unicode codepoint isn't identical to the keyCode, + // then the identifier is likely correct + var codepoint = parseInt(keyIdentifier.substring(unicodePrefixLocation+2), 16); + if (keyCode !== codepoint) + return true; + + // The keyCodes for A-Z and 0-9 are actually identical to their + // Unicode codepoints + if ((keyCode >= 65 && keyCode <= 90) || (keyCode >= 48 && keyCode <= 57)) + return true; + + // The keyIdentifier does NOT appear sane + return false; + + }; + + /** + * Marks a key as pressed, firing the keydown event if registered. Key + * repeat for the pressed key will start after a delay if that key is + * not a modifier. The return value of this function depends on the + * return value of the keydown event handler, if any. + * + * @param {Number} keysym The keysym of the key to press. + * @return {Boolean} true if event should NOT be canceled, false otherwise. + */ + this.press = function(keysym) { + + // Don't bother with pressing the key if the key is unknown + if (keysym === null) return; + + // Only press if released + if (!guac_keyboard.pressed[keysym]) { + + // Mark key as pressed + guac_keyboard.pressed[keysym] = true; + + // Send key event + if (guac_keyboard.onkeydown) { + + var result = guac_keyboard.onkeydown(keysym_to_string[keysym], + modifier_state_to_str()); + last_keydown_result[keysym] = result; + + // Stop any current repeat + window.clearTimeout(key_repeat_timeout); + window.clearInterval(key_repeat_interval); + + // Repeat after a delay as long as pressed + if (!no_repeat[keysym]) + key_repeat_timeout = window.setTimeout(function() { + key_repeat_interval = window.setInterval(function() { + var mods = modifier_state_to_str(); + guac_keyboard.onkeyup(keysym_to_string[keysym], mods); + guac_keyboard.onkeydown(keysym_to_string[keysym], mods); + }, 50); + }, 500); + + return result; + } + } + + // Return the last keydown result by default, resort to false if unknown + return last_keydown_result[keysym] || false; + + }; + + /** + * Marks a key as released, firing the keyup event if registered. + * + * @param {Number} keysym The keysym of the key to release. + */ + this.release = function(keysym) { + + // Only release if pressed + if (guac_keyboard.pressed[keysym]) { + + // Mark key as released + delete guac_keyboard.pressed[keysym]; + + // Stop repeat + window.clearTimeout(key_repeat_timeout); + window.clearInterval(key_repeat_interval); + + // Send key event + if (keysym !== null && guac_keyboard.onkeyup) + guac_keyboard.onkeyup(keysym_to_string[keysym], + modifier_state_to_str()); + + } + + }; + + /** + * Resets the state of this keyboard, releasing all keys, and firing keyup + * events for each released key. + */ + this.reset = function() { + + // Release all pressed keys + for (var keysym in guac_keyboard.pressed) + guac_keyboard.release(parseInt(keysym)); + + // Clear event log + eventLog = []; + + }; + + /** + * Given a keyboard event, updates the local modifier state and remote + * key state based on the modifier flags within the event. This function + * pays no attention to keycodes. + * + * @private + * @param {KeyboardEvent} e + * The keyboard event containing the flags to update. + */ + var update_modifier_state = function update_modifier_state(e) { + + // Get state + var state = Keyboard.ModifierState.fromKeyboardEvent(e); + + // Release alt if implicitly released + if (guac_keyboard.modifiers.alt && state.alt === false) { + guac_keyboard.release(0xFFE9); // Left alt + guac_keyboard.release(0xFFEA); // Right alt + guac_keyboard.release(0xFE03); // AltGr + } + + // Release shift if implicitly released + if (guac_keyboard.modifiers.shift && state.shift === false) { + guac_keyboard.release(0xFFE1); // Left shift + guac_keyboard.release(0xFFE2); // Right shift + } + + // Release ctrl if implicitly released + if (guac_keyboard.modifiers.ctrl && state.ctrl === false) { + guac_keyboard.release(0xFFE3); // Left ctrl + guac_keyboard.release(0xFFE4); // Right ctrl + } + + // Release meta if implicitly released + if (guac_keyboard.modifiers.meta && state.meta === false) { + guac_keyboard.release(0xFFE7); // Left meta + guac_keyboard.release(0xFFE8); // Right meta + } + + // Release hyper if implicitly released + if (guac_keyboard.modifiers.hyper && state.hyper === false) { + guac_keyboard.release(0xFFEB); // Left hyper + guac_keyboard.release(0xFFEC); // Right hyper + } + + // Update state + guac_keyboard.modifiers = state; + + }; + + /** + * Constructs a string representing all currently pressed modifiers. + * + * @return {String} The resulting string. + */ + var modifier_state_to_str = function modifier_state_to_str() { + let masks = [] + if (guac_keyboard.modifiers.alt) masks.push("alt-mask"); + if (guac_keyboard.modifiers.ctrl) masks.push("control-mask"); + if (guac_keyboard.modifiers.meta) masks.push("meta-mask"); + if (guac_keyboard.modifiers.shift) masks.push("shift-mask"); + if (guac_keyboard.modifiers.hyper) masks.push("hyper-mask"); + return masks.join('+') + } + + /** + * Reads through the event log, removing events from the head of the log + * when the corresponding true key presses are known (or as known as they + * can be). + * + * @private + * @return {Boolean} Whether the default action of the latest event should + * be prevented. + */ + function interpret_events() { + + // Do not prevent default if no event could be interpreted + var handled_event = interpret_event(); + if (!handled_event) + return false; + + // Interpret as much as possible + var last_event; + do { + last_event = handled_event; + handled_event = interpret_event(); + } while (handled_event !== null); + + return last_event.defaultPrevented; + + } + + /** + * Releases Ctrl+Alt, if both are currently pressed and the given keysym + * looks like a key that may require AltGr. + * + * @private + * @param {Number} keysym The key that was just pressed. + */ + var release_simulated_altgr = function release_simulated_altgr(keysym) { + + // Both Ctrl+Alt must be pressed if simulated AltGr is in use + if (!guac_keyboard.modifiers.ctrl || !guac_keyboard.modifiers.alt) + return; + + // Assume [A-Z] never require AltGr + if (keysym >= 0x0041 && keysym <= 0x005A) + return; + + // Assume [a-z] never require AltGr + if (keysym >= 0x0061 && keysym <= 0x007A) + return; + + // Release Ctrl+Alt if the keysym is printable + if (keysym <= 0xFF || (keysym & 0xFF000000) === 0x01000000) { + guac_keyboard.release(0xFFE3); // Left ctrl + guac_keyboard.release(0xFFE4); // Right ctrl + guac_keyboard.release(0xFFE9); // Left alt + guac_keyboard.release(0xFFEA); // Right alt + } + + }; + + /** + * Reads through the event log, interpreting the first event, if possible, + * and returning that event. If no events can be interpreted, due to a + * total lack of events or the need for more events, null is returned. Any + * interpreted events are automatically removed from the log. + * + * @private + * @return {KeyEvent} + * The first key event in the log, if it can be interpreted, or null + * otherwise. + */ + var interpret_event = function interpret_event() { + + // Peek at first event in log + var first = eventLog[0]; + if (!first) + return null; + + // Keydown event + if (first instanceof KeydownEvent) { + + var keysym = null; + var accepted_events = []; + + // If event itself is reliable, no need to wait for other events + if (first.reliable) { + keysym = first.keysym; + accepted_events = eventLog.splice(0, 1); + } + + // If keydown is immediately followed by a keypress, use the indicated character + else if (eventLog[1] instanceof KeypressEvent) { + keysym = eventLog[1].keysym; + accepted_events = eventLog.splice(0, 2); + } + + // If keydown is immediately followed by anything else, then no + // keypress can possibly occur to clarify this event, and we must + // handle it now + else if (eventLog[1]) { + keysym = first.keysym; + accepted_events = eventLog.splice(0, 1); + } + + // Fire a key press if valid events were found + if (accepted_events.length > 0) { + + if (keysym) { + + // Fire event + release_simulated_altgr(keysym); + var defaultPrevented = !guac_keyboard.press(keysym); + recentKeysym[first.keyCode] = keysym; + + // If a key is pressed while meta is held down, the keyup will + // never be sent in Chrome, so send it now. (bug #108404) + if (guac_keyboard.modifiers.meta && keysym !== 0xFFE7 && keysym !== 0xFFE8) + guac_keyboard.release(keysym); + + // Record whether default was prevented + for (var i=0; i<accepted_events.length; i++) + accepted_events[i].defaultPrevented = defaultPrevented; + + } + + return first; + + } + + } // end if keydown + + // Keyup event + else if (first instanceof KeyupEvent) { + + // Release specific key if known + var keysym = first.keysym; + if (keysym) { + guac_keyboard.release(keysym); + first.defaultPrevented = true; + } + + // Otherwise, fall back to releasing all keys + else { + guac_keyboard.reset(); + return first; + } + + return eventLog.shift(); + + } // end if keyup + + // Ignore any other type of event (keypress by itself is invalid) + else + return eventLog.shift(); + + // No event interpreted + return null; + + }; + + /** + * Returns the keyboard location of the key associated with the given + * keyboard event. The location differentiates key events which otherwise + * have the same keycode, such as left shift vs. right shift. + * + * @private + * @param {KeyboardEvent} e + * A JavaScript keyboard event, as received through the DOM via a + * "keydown", "keyup", or "keypress" handler. + * + * @returns {Number} + * The location of the key event on the keyboard, as defined at: + * http://www.w3.org/TR/DOM-Level-3-Events/#events-KeyboardEvent + */ + var getEventLocation = function getEventLocation(e) { + + // Use standard location, if possible + if ('location' in e) + return e.location; + + // Failing that, attempt to use deprecated keyLocation + if ('keyLocation' in e) + return e.keyLocation; + + // If no location is available, assume left side + return 0; + + }; + + // When key pressed + element.addEventListener("keydown", function(e) { + + // Only intercept if handler set + if (!guac_keyboard.onkeydown) return; + + var keyCode; + if (window.event) keyCode = window.event.keyCode; + else if (e.which) keyCode = e.which; + + // Fix modifier states + update_modifier_state(e); + + // Ignore (but do not prevent) the "composition" keycode sent by some + // browsers when an IME is in use (see: http://lists.w3.org/Archives/Public/www-dom/2010JulSep/att-0182/keyCode-spec.html) + if (keyCode === 229) + return; + + // Log event + var keydownEvent = new KeydownEvent(keyCode, e.keyIdentifier, e.key, getEventLocation(e)); + eventLog.push(keydownEvent); + + // Interpret as many events as possible, prevent default if indicated + if (interpret_events()) + e.preventDefault(); + + }, true); + + // When key pressed + element.addEventListener("keypress", function(e) { + + // Only intercept if handler set + if (!guac_keyboard.onkeydown && !guac_keyboard.onkeyup) return; + + var charCode; + if (window.event) charCode = window.event.keyCode; + else if (e.which) charCode = e.which; + + // Fix modifier states + update_modifier_state(e); + + // Log event + var keypressEvent = new KeypressEvent(charCode); + eventLog.push(keypressEvent); + + // Interpret as many events as possible, prevent default if indicated + if (interpret_events()) + e.preventDefault(); + + }, true); + + // When key released + element.addEventListener("keyup", function(e) { + + // Only intercept if handler set + if (!guac_keyboard.onkeyup) return; + + e.preventDefault(); + + var keyCode; + if (window.event) keyCode = window.event.keyCode; + else if (e.which) keyCode = e.which; + + // Fix modifier states + update_modifier_state(e); + + // Log event, call for interpretation + var keyupEvent = new KeyupEvent(keyCode, e.keyIdentifier, e.key, getEventLocation(e)); + eventLog.push(keyupEvent); + interpret_events(); + + }, true); + +}; + +/** + * The state of all supported keyboard modifiers. + * @constructor + */ +Keyboard.ModifierState = function() { + + /** + * Whether shift is currently pressed. + * @type {Boolean} + */ + this.shift = false; + + /** + * Whether ctrl is currently pressed. + * @type {Boolean} + */ + this.ctrl = false; + + /** + * Whether alt is currently pressed. + * @type {Boolean} + */ + this.alt = false; + + /** + * Whether meta (apple key) is currently pressed. + * @type {Boolean} + */ + this.meta = false; + + /** + * Whether hyper (windows key) is currently pressed. + * @type {Boolean} + */ + this.hyper = false; + +}; + +/** + * Returns the modifier state applicable to the keyboard event given. + * + * @param {KeyboardEvent} e The keyboard event to read. + * @returns {Keyboard.ModifierState} The current state of keyboard + * modifiers. + */ +Keyboard.ModifierState.fromKeyboardEvent = function(e) { + + var state = new Keyboard.ModifierState(); + + // Assign states from old flags + state.shift = e.shiftKey; + state.ctrl = e.ctrlKey; + state.alt = e.altKey; + state.meta = e.metaKey; + + // Use DOM3 getModifierState() for others + if (e.getModifierState) { + state.hyper = e.getModifierState("OS") + || e.getModifierState("Super") + || e.getModifierState("Hyper") + || e.getModifierState("Win"); + } + + return state; + +}; diff --git a/net/webrtc/www/theme.css b/net/webrtc/www/theme.css new file mode 100644 index 00000000..a79dc14a --- /dev/null +++ b/net/webrtc/www/theme.css @@ -0,0 +1,141 @@ +/* Reset CSS from Eric Meyer */ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} +body { + line-height: 1; +} +ol, ul { + list-style: none; +} +blockquote, q { + quotes: none; +} +blockquote:before, blockquote:after, +q:before, q:after { + content: ''; + content: none; +} +table { + border-collapse: collapse; + border-spacing: 0; +} + +/* Our style */ + +body{ + display: flex; + flex-direction: column; + min-height: 100vh; + background-color: #222; + color: white; +} + +.holygrail-body { + flex: 1 0 auto; + display: flex; +} + +.holygrail-body .content { + width: 100%; +} + +#sessions { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-around; +} + +.holygrail-body .nav { + width: 220px; + list-style: none; + text-align: left; + order: -1; + background-color: #333; + margin: 0; +} + +@media (max-width: 700px) { + .holygrail-body { + flex-direction: column; + } + + .holygrail-body .nav { + width: 100%; + } +} + +.session p span { + float: right; +} + +.session p { + padding-top: 5px; + padding-bottom: 5px; +} + +.stream { + background-color: black; + width: 480px; +} + +#camera-list { + text-align: center; +} + +.button { + border: none; + padding: 8px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; + -webkit-transition-duration: 0.4s; /* Safari */ + transition-duration: 0.4s; + cursor: pointer; + margin: 5px auto; + width: 90%; +} + +.button1 { + background-color: #222; + color: white; + border: 2px solid #4CAF50; + word-wrap: anywhere; +} + +.button1:hover { + background-color: #4CAF50; + color: white; +} + +#image-holder { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-around; +} diff --git a/net/webrtc/www/webrtc.js b/net/webrtc/www/webrtc.js new file mode 100644 index 00000000..eb449d90 --- /dev/null +++ b/net/webrtc/www/webrtc.js @@ -0,0 +1,466 @@ +/* vim: set sts=4 sw=4 et : + * + * Demo Javascript app for negotiating and streaming a sendrecv webrtc stream + * with a GStreamer app. Runs only in passive mode, i.e., responds to offers + * with answers, exchanges ICE candidates, and streams. + * + * Author: Nirbheek Chauhan <nirbheek@centricular.com> + */ + +// Set this to override the automatic detection in websocketServerConnect() +var ws_server; +var ws_port; +// Override with your own STUN servers if you want +var rtc_configuration = {iceServers: [{urls: "stun:stun.l.google.com:19302"}, + /* TODO: do not keep these static and in clear text in production, + * and instead use one of the mechanisms discussed in + * https://groups.google.com/forum/#!topic/discuss-webrtc/nn8b6UboqRA + */ + {'urls': 'turn:turn.homeneural.net:3478?transport=udp', + 'credential': '1qaz2wsx', + 'username': 'test' + }], + /* Uncomment the following line to ensure the turn server is used + * while testing. This should be kept commented out in production, + * as non-relay ice candidates should be preferred + */ + // iceTransportPolicy: "relay", + }; + +var sessions = {} + +/* https://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript */ +function getOurId() { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +} + +function Uint8ToString(u8a){ + var CHUNK_SZ = 0x8000; + var c = []; + for (var i=0; i < u8a.length; i+=CHUNK_SZ) { + c.push(String.fromCharCode.apply(null, u8a.subarray(i, i+CHUNK_SZ))); + } + return c.join(""); +} + +function Session(our_id, peer_id, closed_callback) { + this.id = null; + this.peer_connection = null; + this.ws_conn = null; + this.peer_id = peer_id; + this.our_id = our_id; + this.closed_callback = closed_callback; + this.data_channel = null; + this.input = null; + + this.getVideoElement = function() { + return document.getElementById("stream-" + this.our_id); + }; + + this.resetState = function() { + if (this.peer_connection) { + this.peer_connection.close(); + this.peer_connection = null; + } + var videoElement = this.getVideoElement(); + if (videoElement) { + videoElement.pause(); + videoElement.src = ""; + } + + var session_div = document.getElementById("session-" + this.our_id); + if (session_div) { + session_div.parentNode.removeChild(session_div); + } + if (this.ws_conn) { + this.ws_conn.close(); + this.ws_conn = null; + } + + this.input && this.input.detach(); + this.data_channel = null; + }; + + this.handleIncomingError = function(error) { + this.resetState(); + this.closed_callback(this.our_id); + }; + + this.setStatus = function(text) { + console.log(text); + var span = document.getElementById("status-" + this.our_id); + // Don't set the status if it already contains an error + if (!span.classList.contains('error')) + span.textContent = text; + }; + + this.setError = function(text) { + console.error(text); + var span = document.getElementById("status-" + this.our_id); + span.textContent = text; + span.classList.add('error'); + this.resetState(); + this.closed_callback(this.our_id); + }; + + // Local description was set, send it to peer + this.onLocalDescription = function(desc) { + console.log("Got local description: " + JSON.stringify(desc), this); + var thiz = this; + this.peer_connection.setLocalDescription(desc).then(() => { + this.setStatus("Sending SDP answer"); + var sdp = { + 'type': 'peer', + 'sessionId': this.id, + 'sdp': this.peer_connection.localDescription.toJSON() + }; + this.ws_conn.send(JSON.stringify(sdp)); + }).catch(function(e) { + thiz.setError(e); + }); + }; + + this.onRemoteDescriptionSet = function() { + this.setStatus("Remote SDP set"); + this.setStatus("Got SDP offer"); + this.peer_connection.createAnswer() + .then(this.onLocalDescription.bind(this)).catch(this.setError); + } + + // SDP offer received from peer, set remote description and create an answer + this.onIncomingSDP = function(sdp) { + var thiz = this; + this.peer_connection.setRemoteDescription(sdp) + .then(this.onRemoteDescriptionSet.bind(this)) + .catch(function(e) { + thiz.setError(e) + }); + }; + + // ICE candidate received from peer, add it to the peer connection + this.onIncomingICE = function(ice) { + var candidate = new RTCIceCandidate(ice); + var thiz = this; + this.peer_connection.addIceCandidate(candidate).catch(function(e) { + thiz.setError(e) + }); + }; + + this.onServerMessage = function(event) { + console.log("Received " + event.data); + try { + msg = JSON.parse(event.data); + } catch (e) { + if (e instanceof SyntaxError) { + this.handleIncomingError("Error parsing incoming JSON: " + event.data); + } else { + this.handleIncomingError("Unknown error parsing response: " + event.data); + } + return; + } + + if (msg.type == "registered") { + this.setStatus("Registered with server"); + this.connectPeer(); + } else if (msg.type == "sessionStarted") { + this.setStatus("Registered with server"); + this.id = msg.sessionId; + } else if (msg.type == "error") { + this.handleIncomingError(msg.details); + } else if (msg.type == "endSession") { + this.resetState(); + this.closed_callback(this.our_id); + } else if (msg.type == "peer") { + // Incoming peer message signals the beginning of a call + if (!this.peer_connection) + this.createCall(msg); + + if (msg.sdp != null) { + this.onIncomingSDP(msg.sdp); + } else if (msg.ice != null) { + this.onIncomingICE(msg.ice); + } else { + this.handleIncomingError("Unknown incoming JSON: " + msg); + } + } + }; + + this.streamIsPlaying = function(e) { + this.setStatus("Streaming"); + }; + + this.onServerClose = function(event) { + this.resetState(); + this.closed_callback(this.our_id); + }; + + this.onServerError = function(event) { + this.handleIncomingError('Server error'); + }; + + this.websocketServerConnect = function() { + // Clear errors in the status span + var span = document.getElementById("status-" + this.our_id); + span.classList.remove('error'); + span.textContent = ''; + console.log("Our ID:", this.our_id); + var ws_port = ws_port || '8443'; + if (window.location.protocol.startsWith ("file")) { + var ws_server = ws_server || "127.0.0.1"; + } else if (window.location.protocol.startsWith ("http")) { + var ws_server = ws_server || window.location.hostname; + } else { + throw new Error ("Don't know how to connect to the signalling server with uri" + window.location); + } + var ws_url = 'ws://' + ws_server + ':' + ws_port + this.setStatus("Connecting to server " + ws_url); + this.ws_conn = new WebSocket(ws_url); + /* When connected, immediately register with the server */ + this.ws_conn.addEventListener('open', (event) => { + this.setStatus("Connecting to the peer"); + this.connectPeer(); + }); + this.ws_conn.addEventListener('error', this.onServerError.bind(this)); + this.ws_conn.addEventListener('message', this.onServerMessage.bind(this)); + this.ws_conn.addEventListener('close', this.onServerClose.bind(this)); + }; + + this.connectPeer = function() { + this.setStatus("Connecting " + this.peer_id); + + this.ws_conn.send(JSON.stringify({ + "type": "startSession", + "peerId": this.peer_id + })); + }; + + this.onRemoteStreamAdded = function(event) { + var videoTracks = event.stream.getVideoTracks(); + var audioTracks = event.stream.getAudioTracks(); + + console.log(videoTracks); + + if (videoTracks.length > 0) { + console.log('Incoming stream: ' + videoTracks.length + ' video tracks and ' + audioTracks.length + ' audio tracks'); + this.getVideoElement().srcObject = event.stream; + this.getVideoElement().play(); + } else { + this.handleIncomingError('Stream with unknown tracks added, resetting'); + } + }; + + this.createCall = function(msg) { + console.log('Creating RTCPeerConnection'); + + this.peer_connection = new RTCPeerConnection(rtc_configuration); + this.peer_connection.onaddstream = this.onRemoteStreamAdded.bind(this); + + this.peer_connection.ondatachannel = (event) => { + console.log(`Data channel created: ${event.channel.label}`); + this.data_channel = event.channel; + + video_element = this.getVideoElement(); + if (video_element) { + this.input = new Input(video_element, (data) => { + if (this.data_channel) { + console.log(`Navigation data: ${data}`); + this.data_channel.send(JSON.stringify(data)); + } + }); + } + + this.data_channel.onopen = (event) => { + console.log("Receive channel opened, attaching input"); + this.input.attach(); + } + this.data_channel.onclose = (event) => { + console.info("Receive channel closed"); + this.input && this.input.detach(); + this.data_channel = null; + } + this.data_channel.onerror = (event) => { + this.input && this.input.detach(); + console.warn("Error on receive channel", event.data); + this.data_channel = null; + } + + let buffer = []; + this.data_channel.onmessage = (event) => { + if (typeof event.data === 'string' || event.data instanceof String) { + if (event.data == 'BEGIN_IMAGE') + buffer = []; + else if (event.data == 'END_IMAGE') { + var decoder = new TextDecoder("ascii"); + var array_buffer = new Uint8Array(buffer); + var str = decoder.decode(array_buffer); + let img = document.getElementById("image"); + img.src = 'data:image/png;base64, ' + str; + } + } else { + var i, len = buffer.length + var view = new DataView(event.data); + for (i = 0; i < view.byteLength; i++) { + buffer[len + i] = view.getUint8(i); + } + } + } + } + + this.peer_connection.onicecandidate = (event) => { + if (event.candidate == null) { + console.log("ICE Candidate was null, done"); + return; + } + this.ws_conn.send(JSON.stringify({ + "type": "peer", + "sessionId": this.id, + "ice": event.candidate.toJSON() + })); + }; + + this.setStatus("Created peer connection for call, waiting for SDP"); + }; + + document.getElementById("stream-" + this.our_id).addEventListener("playing", this.streamIsPlaying.bind(this), false); + + this.websocketServerConnect(); +} + +function startSession() { + var peer_id = document.getElementById("camera-id").value; + + if (peer_id === "") { + return; + } + + sessions[peer_id] = new Session(peer_id); +} + +function session_closed(peer_id) { + sessions[peer_id] = null; +} + +function addPeer(peer_id, meta) { + console.log("Meta: ", JSON.stringify(meta)); + + var nav_ul = document.getElementById("camera-list"); + + meta = meta ? meta : {"display-name": peer_id}; + let display_html = `${meta["display-name"] ? meta["display-name"] : peer_id}<ul>`; + for (const key in meta) { + if (key != "display-name") { + display_html += `<li>- ${key}: ${meta[key]}</li>`; + } + } + display_html += "</ul>" + + var li_str = '<li id="peer-' + peer_id + '"><button class="button button1">' + display_html + '</button></li>'; + + nav_ul.insertAdjacentHTML('beforeend', li_str); + var li = document.getElementById("peer-" + peer_id); + li.onclick = function(e) { + var sessions_div = document.getElementById('sessions'); + var our_id = getOurId(); + var session_div_str = '<div class="session" id="session-' + our_id + '"><video preload="none" class="stream" id="stream-' + our_id + '"></video><p>Status: <span id="status-' + our_id + '">unknown</span></p></div>' + sessions_div.insertAdjacentHTML('beforeend', session_div_str); + sessions[peer_id] = new Session(our_id, peer_id, session_closed); + } +} + +function clearPeers() { + var nav_ul = document.getElementById("camera-list"); + + while (nav_ul.firstChild) { + nav_ul.removeChild(nav_ul.firstChild); + } +} + +function onServerMessage(event) { + console.log("Received " + event.data); + + try { + msg = JSON.parse(event.data); + } catch (e) { + if (e instanceof SyntaxError) { + console.error("Error parsing incoming JSON: " + event.data); + } else { + console.error("Unknown error parsing response: " + event.data); + } + return; + } + + if (msg.type == "welcome") { + console.info(`Got welcomed with ID ${msg.peer_id}`); + ws_conn.send(JSON.stringify({ + "type": "list" + })); + } else if (msg.type == "list") { + clearPeers(); + for (i = 0; i < msg.producers.length; i++) { + addPeer(msg.producers[i].id, msg.producers[i].meta); + } + } else if (msg.type == "peerStatusChanged") { + var li = document.getElementById("peer-" + msg.peerId); + if (msg.roles.includes("producer")) { + if (li == null) { + console.error('Adding peer'); + addPeer(msg.peerId, msg.meta); + } + } else if (li != null) { + li.parentNode.removeChild(li); + } + } else { + console.error("Unsupported message: ", msg); + } +}; + +function clearConnection() { + ws_conn.removeEventListener('error', onServerError); + ws_conn.removeEventListener('message', onServerMessage); + ws_conn.removeEventListener('close', onServerClose); + ws_conn = null; +} + +function onServerClose(event) { + clearConnection(); + clearPeers(); + console.log("Close"); + window.setTimeout(connect, 1000); +}; + +function onServerError(event) { + clearConnection(); + clearPeers(); + console.log("Error", event); + window.setTimeout(connect, 1000); +}; + +function connect() { + var ws_port = ws_port || '8443'; + if (window.location.protocol.startsWith ("file")) { + var ws_server = ws_server || "127.0.0.1"; + } else if (window.location.protocol.startsWith ("http")) { + var ws_server = ws_server || window.location.hostname; + } else { + throw new Error ("Don't know how to connect to the signalling server with uri" + window.location); + } + var ws_url = 'ws://' + ws_server + ':' + ws_port + console.log("Connecting listener"); + ws_conn = new WebSocket(ws_url); + ws_conn.addEventListener('open', (event) => { + ws_conn.send(JSON.stringify({ + "type": "setPeerStatus", + "roles": ["listener"] + })); + }); + ws_conn.addEventListener('error', onServerError); + ws_conn.addEventListener('message', onServerMessage); + ws_conn.addEventListener('close', onServerClose); +} + +function setup() { + connect(); +} |
