diff --git a/.gitignore b/.gitignore index 613083218..e5e7c1e51 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target *.css.map .sass-cache .vagrant +.rustwide diff --git a/Cargo.lock b/Cargo.lock index 89e56cff8..63b42cf7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,7 +74,7 @@ name = "atty" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -91,7 +91,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -102,7 +102,7 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -131,6 +131,14 @@ dependencies = [ "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "0.7.0" @@ -248,7 +256,7 @@ dependencies = [ "jobserver 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -290,7 +298,7 @@ name = "chrono" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", @@ -332,7 +340,7 @@ name = "commoncrypto-sys" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -369,7 +377,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -378,7 +386,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -386,7 +394,7 @@ name = "core-foundation-sys" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -436,7 +444,7 @@ dependencies = [ "html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)", "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "magic 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", "params 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -451,6 +459,7 @@ dependencies = [ "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "rustwide 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -545,7 +554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "curl-sys 0.4.18 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", "schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -559,7 +568,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libnghttp2-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -581,7 +590,7 @@ name = "dirs" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -666,7 +675,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -676,7 +685,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -685,7 +694,7 @@ name = "flate2" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -713,7 +722,7 @@ name = "fs2" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -776,13 +785,23 @@ dependencies = [ "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "getrandom" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "git2" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1028,7 +1047,7 @@ name = "iovec" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1064,7 +1083,7 @@ name = "jobserver" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1100,7 +1119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.43" +version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1119,7 +1138,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libssh2-sys 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1140,7 +1159,7 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1153,7 +1172,7 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1194,7 +1213,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "magic-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1203,7 +1222,7 @@ name = "magic-sys" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1236,7 +1255,7 @@ name = "memchr" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1244,7 +1263,7 @@ name = "memchr" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1301,7 +1320,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1314,7 +1333,7 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1339,7 +1358,7 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1397,7 +1416,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1413,7 +1432,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1425,6 +1444,18 @@ dependencies = [ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nix" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nodrop" version = "0.1.12" @@ -1517,7 +1548,7 @@ name = "num_cpus" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1537,7 +1568,7 @@ dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.45 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1553,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1603,7 +1634,7 @@ name = "parking_lot_core" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1615,6 +1646,11 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "persistent" version = "0.3.0" @@ -1821,7 +1857,7 @@ version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1831,7 +1867,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1842,7 +1878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1853,7 +1889,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1913,7 +1949,7 @@ name = "rand_jitter" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1925,7 +1961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2022,7 +2058,7 @@ dependencies = [ [[package]] name = "remove_dir_all" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2173,6 +2209,35 @@ dependencies = [ "stb_truetype 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rustwide" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-process 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ryu" version = "0.2.6" @@ -2201,7 +2266,7 @@ name = "sass-rs" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "sass-sys 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2211,7 +2276,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2259,6 +2324,11 @@ name = "scopeguard" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "security-framework" version = "0.2.1" @@ -2266,7 +2336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2276,7 +2346,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2387,7 +2457,7 @@ name = "signal-hook" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook-registry 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2397,7 +2467,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2440,7 +2510,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2590,7 +2660,7 @@ dependencies = [ "bytesize 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2602,8 +2672,9 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2612,7 +2683,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2621,10 +2692,10 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2651,7 +2722,7 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2677,7 +2748,7 @@ name = "time" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2756,7 +2827,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2788,7 +2859,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2858,7 +2929,7 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3108,6 +3179,11 @@ dependencies = [ "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi" version = "0.2.8" @@ -3163,6 +3239,14 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "xml-rs" version = "0.7.0" @@ -3186,6 +3270,7 @@ dependencies = [ "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" @@ -3255,6 +3340,7 @@ dependencies = [ "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" "checksum fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "34dd4c507af68d37ffef962063dfa1944ce0dd4d5b82043dbab1dabe088610c3" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum getrandom 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "473a1265acc8ff1e808cd0a1af8cee3c2ee5200916058a2ca113c29f2d903571" "checksum git2 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "924b2e7d2986e625dcad89e8a429a7b3adee3c3d71e585f4a66c4f7e78715e31" "checksum git2-curl 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f21f0550fd5d3f7c5adb94797fcd3d1002d7fc1fa349c82fe44f3c97ef80b62c" "checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" @@ -3287,7 +3373,7 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" "checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0" -"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" +"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" "checksum libflate 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "21138fc6669f438ed7ae3559d5789a5f0ba32f28c1f0608d1e452b0bb06ee936" "checksum libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "941a41e23f77323b8c9d2ee118aec9ee39dfc176078c18b4757d3bad049d9ff7" "checksum libnghttp2-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ffbfb81475cc9f625e44f3a8f8b9cf7173815ae1c7cc2fa91853ec009e38198" @@ -3322,6 +3408,7 @@ dependencies = [ "checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" +"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" "checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" @@ -3343,6 +3430,7 @@ dependencies = [ "checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5" "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" "checksum persistent 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c9c94f2ef72dc272c6bcc8157ccf2bc7da14f4c58c69059ac2fc48492d6916" "checksum pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0a6dda33d67c26f0aac90d324ab2eb7239c819fc7b2552fe9faa4fe88441edc8" "checksum pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" @@ -3388,7 +3476,7 @@ dependencies = [ "checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" "checksum regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d76410686f9e3a17f06128962e0ecc5755870bb890c34820c7af7f1db2e1d48" -"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1d68c7bf0b1dc3860b80c6d31d05808bf54cdc1bfc70a4680893791becd083ae" "checksum route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" "checksum router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b1797ff166029cb632237bb5542696e54961b4cf75a324c6f05c9cf0584e4e" @@ -3401,6 +3489,7 @@ dependencies = [ "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustfix 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "af7c21531a91512a4a51b490be6ba1c8eff34fdda0dc5bf87dc28d86748aac56" "checksum rusttype 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b8eb11f5b0a98c8eca2fb1483f42646d8c340e83e46ab416f8a063a0fd0eeb20" +"checksum rustwide 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d62fdc88c46d136aea71a9bd56503cb7570f8e11086d46ddd97a53a45ab8612c" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" @@ -3413,6 +3502,7 @@ dependencies = [ "checksum schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9a69defe7b625fa5c4bfda0a1525c9729baef68f620e505464b7bf0a4d1697f6" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" "checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" @@ -3512,6 +3602,7 @@ dependencies = [ "checksum walkdir 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c66c0b9792f0a765345452775f3adbd28dde9d33f30d13e5dcc5ae17cf6f3780" "checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" "checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" +"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" @@ -3520,4 +3611,5 @@ dependencies = [ "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" "checksum xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c1cb601d29fe2c2ac60a2b2e5e293994d87a1f6fa9687a31a15270f909be9c2" diff --git a/Cargo.toml b/Cargo.toml index fffbc96b4..aebb4e804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,8 @@ tokio = "0.1" systemstat = "0.1.4" prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" +rustwide = "0.3.0" +tempdir = "0.3" # iron dependencies iron = "0.5" @@ -51,9 +53,6 @@ staticfile = { version = "0.4", features = [ "cache" ] } version = "0.15" features = [ "with-time", "with-rustc-serialize" ] -[dev-dependencies] -tempdir = "0.3" - [build-dependencies] time = "0.1" git2 = "0.9" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 71e6193b9..b048c9b82 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -5,14 +5,15 @@ extern crate clap; extern crate log; extern crate env_logger; extern crate time; +extern crate rustwide; use std::env; use std::path::PathBuf; use clap::{Arg, App, SubCommand}; -use cratesfyi::{DocBuilder, DocBuilderOptions, db}; -use cratesfyi::utils::{build_doc, add_crate_to_queue}; +use cratesfyi::{DocBuilder, RustwideBuilder, DocBuilderOptions, db}; +use cratesfyi::utils::add_crate_to_queue; use cratesfyi::start_web_server; use cratesfyi::db::{add_path_into_database, connect_db}; @@ -23,20 +24,6 @@ pub fn main() { let matches = App::new("cratesfyi") .version(cratesfyi::BUILD_VERSION) .about(env!("CARGO_PKG_DESCRIPTION")) - .subcommand(SubCommand::with_name("doc") - .about("Builds documentation of a crate") - .arg(Arg::with_name("CRATE_NAME") - .index(1) - .required(true) - .help("Crate name")) - .arg(Arg::with_name("CRATE_VERSION") - .index(2) - .required(false) - .help("Crate version")) - .arg(Arg::with_name("TARGET") - .index(3) - .required(false) - .help("The target platform to compile for"))) .subcommand(SubCommand::with_name("build") .about("Builds documentation in a chroot environment") .arg(Arg::with_name("PREFIX") @@ -48,21 +35,6 @@ pub fn main() { .long("destination") .help("Sets destination path") .takes_value(true)) - .arg(Arg::with_name("CHROOT_PATH") - .short("c") - .long("chroot-path") - .help("Sets chroot path") - .takes_value(true)) - .arg(Arg::with_name("CHROOT_USER") - .short("u") - .long("chroot-user") - .help("Sets chroot user name") - .takes_value(true)) - .arg(Arg::with_name("CONTAINER_NAME") - .short("n") - .long("container-name") - .help("Sets name of the container") - .takes_value(true)) .arg(Arg::with_name("CRATES_IO_INDEX_PATH") .long("crates-io-index-path") .help("Sets crates.io-index path") @@ -152,15 +124,7 @@ pub fn main() { - // doc subcommand - if let Some(matches) = matches.subcommand_matches("doc") { - let name = matches.value_of("CRATE_NAME").unwrap(); - let version = matches.value_of("CRATE_VERSION"); - let target = matches.value_of("TARGET"); - if let Err(e) = build_doc(name, version, target) { - panic!("{:#?}", e); - } - } else if let Some(matches) = matches.subcommand_matches("build") { + if let Some(matches) = matches.subcommand_matches("build") { let docbuilder_opts = { let mut docbuilder_opts = if let Some(prefix) = matches.value_of("PREFIX") { DocBuilderOptions::from_prefix(PathBuf::from(prefix)) @@ -175,18 +139,6 @@ pub fn main() { docbuilder_opts.destination = PathBuf::from(destination); } - if let Some(chroot_path) = matches.value_of("CHROOT_PATH") { - docbuilder_opts.chroot_path = PathBuf::from(chroot_path); - } - - if let Some(chroot_user) = matches.value_of("CHROOT_USER") { - docbuilder_opts.chroot_user = chroot_user.to_string(); - } - - if let Some(container_name) = matches.value_of("CONTAINER_NAME") { - docbuilder_opts.container_name = container_name.to_string(); - } - if let Some(crates_io_index_path) = matches.value_of("CRATES_IO_INDEX_PATH") { docbuilder_opts.crates_io_index_path = PathBuf::from(crates_io_index_path); } @@ -204,16 +156,19 @@ pub fn main() { if let Some(_) = matches.subcommand_matches("world") { docbuilder.load_cache().expect("Failed to load cache"); - docbuilder.build_world().expect("Failed to build world"); + let mut builder = RustwideBuilder::init().unwrap(); + builder.build_world(&mut docbuilder).expect("Failed to build world"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(matches) = matches.subcommand_matches("crate") { docbuilder.load_cache().expect("Failed to load cache"); - docbuilder.build_package(matches.value_of("CRATE_NAME").unwrap(), - matches.value_of("CRATE_VERSION").unwrap()) + let mut builder = RustwideBuilder::init().unwrap(); + builder.build_package(&mut docbuilder, matches.value_of("CRATE_NAME").unwrap(), + matches.value_of("CRATE_VERSION").unwrap()) .expect("Building documentation failed"); docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(_) = matches.subcommand_matches("add-essential-files") { - docbuilder.add_essential_files().expect("Failed to add essential files"); + let builder = RustwideBuilder::init().unwrap(); + builder.add_essential_files().expect("failed to add essential files"); } else if let Some(_) = matches.subcommand_matches("lock") { docbuilder.lock().expect("Failed to lock"); } else if let Some(_) = matches.subcommand_matches("unlock") { @@ -287,5 +242,6 @@ fn logger_init() { record.args()) }); builder.parse(&env::var("RUST_LOG").unwrap_or("cratesfyi=info".to_owned())); - builder.init(); + + rustwide::logging::init_with(builder.build()); } diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 7def3504a..08dc83fd6 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,21 +1,19 @@ -use ChrootBuilderResult; use Metadata; -use utils::source_path; +use utils::MetadataPackage; +use docbuilder::BuildResult; use regex::Regex; use std::io::prelude::*; use std::io::BufReader; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::fs; -use cargo::core::{Package, TargetKind}; -use cargo::core::dependency::Kind; use rustc_serialize::json::{Json, ToJson}; use slug::slugify; use reqwest::Client; use reqwest::header::ACCEPT; -use semver; +use semver::Version; use postgres::Connection; use time; use error::Result; @@ -24,27 +22,30 @@ use failure::err_msg; /// Adds a package into database. /// /// Package must be built first. -pub fn add_package_into_database(conn: &Connection, - pkg: &Package, - res: &ChrootBuilderResult, +pub(crate) fn add_package_into_database(conn: &Connection, + metadata_pkg: &MetadataPackage, + source_dir: &Path, + res: &BuildResult, files: Option, - doc_targets: Vec) + doc_targets: Vec, + has_docs: bool, + has_examples: bool) -> Result { debug!("Adding package into database"); - let crate_id = try!(initialize_package_in_database(&conn, &pkg)); - let dependencies = convert_dependencies(&pkg); - let rustdoc = get_rustdoc(&pkg).unwrap_or(None); - let readme = get_readme(&pkg).unwrap_or(None); - let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(&pkg)); - let is_library = match pkg.targets()[0].kind() { - &TargetKind::Lib(_) => true, + let crate_id = try!(initialize_package_in_database(&conn, metadata_pkg)); + let dependencies = convert_dependencies(metadata_pkg); + let rustdoc = get_rustdoc(metadata_pkg, source_dir).unwrap_or(None); + let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None); + let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(metadata_pkg)); + let is_library = match metadata_pkg.targets[0].kind.as_slice() { + &[ref kind] if kind == "lib" => true, _ => false, }; - let metadata = Metadata::from_package(pkg)?; + let metadata = Metadata::from_source_dir(source_dir)?; let release_id: i32 = { let rows = try!(conn.query("SELECT id FROM releases WHERE crate_id = $1 AND version = $2", - &[&crate_id, &format!("{}", pkg.manifest().version())])); + &[&crate_id, &format!("{}", metadata_pkg.version)])); if rows.len() == 0 { let rows = try!(conn.query("INSERT INTO releases ( @@ -62,29 +63,29 @@ pub fn add_package_into_database(conn: &Connection, ) RETURNING id", &[&crate_id, - &format!("{}", pkg.manifest().version()), + &metadata_pkg.version, &release_time, &dependencies.to_json(), - &pkg.targets()[0].name().replace("-", "_"), + &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, - &res.build_success, - &res.have_doc, + &res.successful, + &has_docs, &false, // TODO: Add test status somehow - &pkg.manifest().metadata().license, - &pkg.manifest().metadata().repository, - &pkg.manifest().metadata().homepage, - &pkg.manifest().metadata().description, + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, &rustdoc, &readme, - &pkg.manifest().metadata().authors.to_json(), - &pkg.manifest().metadata().keywords.to_json(), - &res.have_examples, + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), + &has_examples, &downloads, &files, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation, + &metadata_pkg.documentation, &metadata.default_target])); // return id rows.get(0).get(0) @@ -116,42 +117,43 @@ pub fn add_package_into_database(conn: &Connection, default_target = $25 WHERE crate_id = $1 AND version = $2", &[&crate_id, - &format!("{}", pkg.manifest().version()), + &format!("{}", metadata_pkg.version), &release_time, &dependencies.to_json(), - &pkg.targets()[0].name().replace("-", "_"), + &metadata_pkg.targets[0].name.replace("-", "_"), &yanked, - &res.build_success, - &res.have_doc, + &res.successful, + &has_docs, &false, // TODO: Add test status somehow - &pkg.manifest().metadata().license, - &pkg.manifest().metadata().repository, - &pkg.manifest().metadata().homepage, - &pkg.manifest().metadata().description, + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, &rustdoc, &readme, - &pkg.manifest().metadata().authors.to_json(), - &pkg.manifest().metadata().keywords.to_json(), - &res.have_examples, + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), + &has_examples, &downloads, &files, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation, + &metadata_pkg.documentation, &metadata.default_target])); rows.get(0).get(0) } }; - try!(add_keywords_into_database(&conn, &pkg, &release_id)); - try!(add_authors_into_database(&conn, &pkg, &release_id)); - try!(add_owners_into_database(&conn, &pkg, &crate_id)); + try!(add_keywords_into_database(&conn, &metadata_pkg, &release_id)); + try!(add_authors_into_database(&conn, &metadata_pkg, &release_id)); + try!(add_owners_into_database(&conn, &metadata_pkg, &crate_id)); // Update versions { + let metadata_version = Version::parse(&metadata_pkg.version)?; let mut versions: Json = try!(conn.query("SELECT versions FROM crates WHERE id = $1", &[&crate_id])) .get(0) @@ -159,13 +161,13 @@ pub fn add_package_into_database(conn: &Connection, if let Some(versions_array) = versions.as_array_mut() { let mut found = false; for version in versions_array.clone() { - if &semver::Version::parse(version.as_string().unwrap()).unwrap() == - pkg.manifest().version() { + let version = Version::parse(version.as_string().unwrap())?; + if version != metadata_version { found = true; } } if !found { - versions_array.push(format!("{}", &pkg.manifest().version()).to_json()); + versions_array.push(format!("{}", &metadata_pkg.version).to_json()); } } let _ = conn.query("UPDATE crates SET versions = $1 WHERE id = $2", @@ -177,9 +179,9 @@ pub fn add_package_into_database(conn: &Connection, /// Adds a build into database -pub fn add_build_into_database(conn: &Connection, +pub(crate) fn add_build_into_database(conn: &Connection, release_id: &i32, - res: &ChrootBuilderResult) + res: &BuildResult) -> Result { debug!("Adding build into database"); let rows = try!(conn.query("INSERT INTO builds (rid, rustc_version, @@ -189,20 +191,19 @@ pub fn add_build_into_database(conn: &Connection, RETURNING id", &[release_id, &res.rustc_version, - &res.cratesfyi_version, - &res.build_success, - &res.output])); + &res.docsrs_version, + &res.successful, + &res.build_log])); Ok(rows.get(0).get(0)) } -fn initialize_package_in_database(conn: &Connection, pkg: &Package) -> Result { - let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1", - &[&pkg.manifest().name().as_str()])); +fn initialize_package_in_database(conn: &Connection, pkg: &MetadataPackage) -> Result { + let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1", &[&pkg.name])); // insert crate into database if it is not exists if rows.len() == 0 { rows = try!(conn.query("INSERT INTO crates (name) VALUES ($1) RETURNING id", - &[&pkg.manifest().name().as_str()])); + &[&pkg.name])); } Ok(rows.get(0).get(0)) } @@ -210,16 +211,12 @@ fn initialize_package_in_database(conn: &Connection, pkg: &Package) -> Result -fn convert_dependencies(pkg: &Package) -> Vec<(String, String, String)> { +fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> { let mut dependencies: Vec<(String, String, String)> = Vec::new(); - for dependency in pkg.manifest().dependencies() { - let name = dependency.package_name().to_string(); - let version = format!("{}", dependency.version_req()); - let kind = match dependency.kind() { - Kind::Normal => "normal", - Kind::Development => "dev", - Kind::Build => "build", - }; + for dependency in &pkg.dependencies { + let name = dependency.name.clone(); + let version = dependency.req.clone(); + let kind = dependency.kind.clone().unwrap_or_else(|| "normal".into()); dependencies.push((name, version, kind.to_string())); } dependencies @@ -227,9 +224,8 @@ fn convert_dependencies(pkg: &Package) -> Vec<(String, String, String)> { /// Reads readme if there is any read defined in Cargo.toml of a Package -fn get_readme(pkg: &Package) -> Result> { - let readme_path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found")))) - .join(pkg.manifest().metadata().readme.clone().unwrap_or("README.md".to_owned())); +fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result> { + let readme_path = source_dir.join(pkg.readme.clone().unwrap_or("README.md".to_owned())); if !readme_path.exists() { return Ok(None); @@ -249,14 +245,13 @@ fn get_readme(pkg: &Package) -> Result> { } -fn get_rustdoc(pkg: &Package) -> Result> { - if let Some(src_path) = pkg.manifest().targets()[0].src_path().path() { +fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result> { + if let Some(src_path) = &pkg.targets[0].src_path { + let src_path = Path::new(src_path); if src_path.is_absolute() { read_rust_doc(src_path) } else { - let mut path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found")))); - path.push(src_path); - read_rust_doc(path.as_path()) + read_rust_doc(&source_dir.join(src_path)) } } else { // FIXME: should we care about metabuild targets? @@ -294,11 +289,10 @@ fn read_rust_doc(file_path: &Path) -> Result> { /// Get release_time, yanked and downloads from crates.io -fn get_release_time_yanked_downloads - (pkg: &Package) - -> Result<(Option, Option, Option)> { - let url = format!("https://crates.io/api/v1/crates/{}/versions", - pkg.manifest().name()); +fn get_release_time_yanked_downloads( + pkg: &MetadataPackage, +) -> Result<(Option, Option, Option)> { + let url = format!("https://crates.io/api/v1/crates/{}/versions", pkg.name); // FIXME: There is probably better way to do this // and so many unwraps... let client = Client::new(); @@ -321,7 +315,7 @@ fn get_release_time_yanked_downloads .and_then(|v| v.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))); - if &semver::Version::parse(version_num).unwrap() == pkg.manifest().version() { + if semver::Version::parse(version_num).unwrap().to_string() == pkg.version { let release_time_raw = try!(version.get("created_at") .and_then(|c| c.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))); @@ -346,8 +340,8 @@ fn get_release_time_yanked_downloads /// Adds keywords into database -fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> { - for keyword in &pkg.manifest().metadata().keywords { +fn add_keywords_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { + for keyword in &pkg.keywords { let slug = slugify(&keyword); let keyword_id: i32 = { let rows = try!(conn.query("SELECT id FROM keywords WHERE slug = $1", &[&slug])); @@ -371,10 +365,10 @@ fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32 /// Adds authors into database -fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> { +fn add_authors_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { let author_capture_re = Regex::new("^([^><]+)<*(.*?)>*$").unwrap(); - for author in &pkg.manifest().metadata().authors { + for author in &pkg.authors { if let Some(author_captures) = author_capture_re.captures(&author[..]) { let author = author_captures.get(1).map(|m| m.as_str()).unwrap_or("").trim(); let email = author_captures.get(2).map(|m| m.as_str()).unwrap_or("").trim(); @@ -405,10 +399,9 @@ fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32) /// Adds owners into database -fn add_owners_into_database(conn: &Connection, pkg: &Package, crate_id: &i32) -> Result<()> { +fn add_owners_into_database(conn: &Connection, pkg: &MetadataPackage, crate_id: &i32) -> Result<()> { // owners available in: https://crates.io/api/v1/crates/rand/owners - let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", - &pkg.manifest().name()); + let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", pkg.name); let client = Client::new(); let mut res = try!(client.get(&owners_url[..]) .header(ACCEPT, "application/json") diff --git a/src/db/migrate.rs b/src/db/migrate.rs index c71a65738..c1adc1f12 100644 --- a/src/db/migrate.rs +++ b/src/db/migrate.rs @@ -179,6 +179,20 @@ pub fn migrate(version: Option) -> CratesfyiResult<()> { // downgrade query "ALTER TABLE queue DROP COLUMN priority;" ), + migration!( + // version + 3, + // description + "Added sandbox_overrides table", + // upgrade query + "CREATE TABLE sandbox_overrides ( + crate_name VARCHAR NOT NULL PRIMARY KEY, + max_memory_bytes INTEGER, + timeout_seconds INTEGER + );", + // downgrade query + "DROP TABLE sandbox_overrides;" + ), ]; for migration in migrations { diff --git a/src/db/mod.rs b/src/db/mod.rs index 7c51c9f48..a83ce902a 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,7 +1,7 @@ //! Database operations -pub use self::add_package::add_package_into_database; -pub use self::add_package::add_build_into_database; +pub(crate) use self::add_package::add_package_into_database; +pub(crate) use self::add_package::add_build_into_database; pub use self::file::add_path_into_database; pub use self::migrate::migrate; diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs deleted file mode 100644 index aea0fe6c5..000000000 --- a/src/docbuilder/chroot_builder.rs +++ /dev/null @@ -1,539 +0,0 @@ - -use super::DocBuilder; -use super::crates::crates_from_path; -use super::metadata::Metadata; -use utils::{get_package, source_path, copy_doc_dir, - update_sources, parse_rustc_version, command_result}; -use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database}; -use cargo::core::Package; -use cargo::util::CargoResultExt; -use std::process::Command; -use std::path::{Path, PathBuf}; -use std::fs::{self, remove_dir_all}; -use postgres::Connection; -use rustc_serialize::json::{Json, ToJson}; -use error::Result; -use systemstat::{Platform, Filesystem, System}; - -const MAX_DISK_USAGE: f32 = 80.0; - -/// List of targets supported by docs.rs -const TARGETS: [&'static str; 6] = [ - "i686-apple-darwin", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" -]; - - - -#[derive(Debug)] -pub struct ChrootBuilderResult { - pub output: String, - pub build_success: bool, - pub have_doc: bool, - pub have_examples: bool, - pub rustc_version: String, - pub cratesfyi_version: String, -} - - -impl DocBuilder { - /// Builds every package documentation in chroot environment - pub fn build_world(&mut self) -> Result<()> { - try!(update_sources()); - - let mut count = 0; - - crates(self.options.crates_io_index_path.clone(), |name, version| { - match self.build_package(name, version) { - Ok(status) => { - count += 1; - if status && count % 10 == 0 { - let _ = self.save_cache(); - } - } - Err(err) => warn!("Failed to build package {}-{}: {}", name, version, err), - } - self.cache.insert(format!("{}-{}", name, version)); - }) - } - - - /// Builds package documentation in chroot environment and adds into cratesfyi database - pub fn build_package(&mut self, name: &str, version: &str) -> Result { - // Skip crates according to options - if (self.options.skip_if_log_exists && - self.cache.contains(&format!("{}-{}", name, version)[..])) || - (self.options.skip_if_exists && - self.db_cache.contains(&format!("{}-{}", name, version)[..])) { - return Ok(false); - } - - info!("Building package {}-{}", name, version); - - // Start with clean documentation directory - try!(self.remove_build_dir()); - - // Database connection - let conn = try!(connect_db()); - - // get_package (and cargo) is using semver, add '=' in front of version. - let pkg = try!(get_package(name, Some(&format!("={}", version)[..]))); - let metadata = Metadata::from_package(&pkg)?; - let res = self.build_package_in_chroot(&pkg, metadata.default_target.clone()); - - // copy sources and documentation - let file_list = try!(self.add_sources_into_database(&conn, &pkg)); - let successfully_targets = if res.have_doc { - try!(self.copy_documentation(&pkg, - &res.rustc_version, - metadata.default_target.as_ref().map(String::as_str), - true)); - let successfully_targets = self.build_package_for_all_targets(&pkg); - for target in &successfully_targets { - try!(self.copy_documentation(&pkg, &res.rustc_version, Some(target), false)); - } - try!(self.add_documentation_into_database(&conn, &pkg)); - successfully_targets - } else { - Vec::new() - }; - - let release_id = try!(add_package_into_database(&conn, - &pkg, - &res, - Some(file_list), - successfully_targets)); - try!(add_build_into_database(&conn, &release_id, &res)); - - // remove documentation, source and build directory after we are done - try!(self.clean(&pkg)); - - // add package into build cache - self.cache.insert(format!("{}-{}", name, version)); - - Ok(res.build_success) - } - - - /// Builds documentation of a package with cratesfyi in chroot environment - fn build_package_in_chroot(&self, package: &Package, default_target: Option) -> ChrootBuilderResult { - debug!("Building package in chroot"); - let (rustc_version, cratesfyi_version) = self.get_versions(); - let cmd = format!("cratesfyi doc {} ={} {}", - package.manifest().name(), - package.manifest().version(), - default_target.as_ref().unwrap_or(&"".to_string())); - match self.chroot_command(cmd) { - Ok(o) => { - ChrootBuilderResult { - output: o, - build_success: true, - have_doc: self.have_documentation(&package, default_target), - have_examples: self.have_examples(&package), - rustc_version: rustc_version, - cratesfyi_version: cratesfyi_version, - } - } - Err(e) => { - ChrootBuilderResult { - output: e.to_string(), - build_success: false, - have_doc: false, - have_examples: self.have_examples(&package), - rustc_version: rustc_version, - cratesfyi_version: cratesfyi_version, - } - } - } - } - - - - /// Builds documentation of crate for every target and returns Vec of successfully targets - fn build_package_for_all_targets(&self, package: &Package) -> Vec { - let mut successfuly_targets = Vec::new(); - - for target in TARGETS.iter() { - debug!("Building {} for {}", canonical_name(&package), target); - let cmd = format!("cratesfyi doc {} ={} {}", - package.manifest().name(), - package.manifest().version(), - target); - if let Ok(_) = self.chroot_command(cmd) { - // Cargo is not giving any error and not generating documentation of some crates - // when we use a target compile options. Check documentation exists before - // adding target to successfully_targets. - // FIXME: Need to figure out why some docs are not generated with target option - let target_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join(&target) - .join("doc"); - if target_doc_path.exists() { - successfuly_targets.push(target.to_string()); - } - } - } - successfuly_targets - } - - - /// Copies documentation to destination directory - fn copy_documentation(&self, - package: &Package, - rustc_version: &str, - target: Option<&str>, - is_default_target: bool) - -> Result<()> { - let mut crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - - // docs are available in cratesfyi/$TARGET when target is being used - if let Some(target) = target { - crate_doc_path.push(target); - } - - let mut destination = PathBuf::from(&self.options.destination) - .join(format!("{}/{}", - package.manifest().name(), - package.manifest().version())); - - // only add target name to destination directory when we are copying a non-default target. - // this is allowing us to host documents in the root of the crate documentation directory. - // for example winapi will be available in docs.rs/winapi/$version/winapi/ for it's - // default target: x86_64-pc-windows-msvc. But since it will be built under - // cratesfyi/x86_64-pc-windows-msvc we still need target in this function. - if !is_default_target { - if let Some(target) = target { - destination.push(target); - } - } - - copy_doc_dir(crate_doc_path, - destination, - parse_rustc_version(rustc_version)?.trim()) - } - - - /// Removes build directory of a package in chroot - fn remove_build_dir(&self) -> Result<()> { - let crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join("doc"); - let _ = remove_dir_all(crate_doc_path); - for target in TARGETS.iter() { - let crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join(target) - .join("doc"); - let _ = remove_dir_all(crate_doc_path); - } - - let crate_build_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - let fs = mount_for_path(&crate_build_path); - let disk_usage = 100.0 - 100.0 * (fs.free.as_usize() as f32 / fs.total.as_usize() as f32); - if disk_usage >= MAX_DISK_USAGE { - info!("Cleaning target directory, disk usage {:.2} exceeded {:.2}", - disk_usage, MAX_DISK_USAGE); - let _ = remove_dir_all(&crate_build_path); - let _ = fs::create_dir_all(&crate_build_path); - } - - Ok(()) - } - - - /// Remove documentation, build directory and sources directory of a package - fn clean(&self, package: &Package) -> Result<()> { - debug!("Cleaning package"); - let documentation_path = PathBuf::from(&self.options.destination) - .join(package.manifest().name().as_str()); - let source_path = source_path(&package).unwrap(); - // Some crates don't have documentation, so we don't care if removing_dir_all fails - let _ = self.remove_build_dir(); - let _ = remove_dir_all(documentation_path); - let _ = remove_dir_all(source_path); - Ok(()) - } - - - /// Runs a command in a chroot environment - fn chroot_command>(&self, cmd: T) -> Result { - command_result(Command::new("sudo") - .arg("lxc-attach") - .arg("-n") - .arg(&self.options.container_name) - .arg("--") - .arg("su") - .arg("-") - .arg(&self.options.chroot_user) - .arg("-c") - .arg(cmd.as_ref()) - .output() - .unwrap()) - } - - - /// Checks a package build directory to determine if package have docs - /// - /// This function is checking first target in targets to see if documentation exists for a - /// crate. Package must be successfully built in chroot environment first. - fn have_documentation(&self, package: &Package, default_target: Option) -> bool { - let mut crate_doc_path = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi"); - - if let Some(default_doc_path) = default_target { - crate_doc_path.push(default_doc_path); - } - - crate_doc_path.push("doc"); - crate_doc_path.push(package.targets()[0].name().replace("-", "_").to_string()); - crate_doc_path.exists() - } - - - /// Checks if package have examples - fn have_examples(&self, package: &Package) -> bool { - let path = source_path(&package).unwrap().join("examples"); - path.exists() && path.is_dir() - } - - - /// Gets rustc and cratesfyi version from chroot environment - pub fn get_versions(&self) -> (String, String) { - // It is safe to use expect here - // chroot environment must always have rustc and cratesfyi installed - (String::from(self.chroot_command("rustc --version") - .expect("Failed to get rustc version") - .trim()), - String::from(self.chroot_command("cratesfyi --version") - .expect("Failed to get cratesfyi version") - .trim())) - } - - - /// Adds sources into database - fn add_sources_into_database(&self, conn: &Connection, package: &Package) -> Result { - debug!("Adding sources into database"); - let prefix = format!("sources/{}/{}", - package.manifest().name(), - package.manifest().version()); - add_path_into_database(conn, &prefix, source_path(&package).unwrap()) - } - - - /// Adds documentations into database - fn add_documentation_into_database(&self, - conn: &Connection, - package: &Package) - -> Result { - debug!("Adding documentation into database"); - let prefix = format!("rustdoc/{}/{}", - package.manifest().name(), - package.manifest().version()); - let crate_doc_path = PathBuf::from(&self.options.destination).join(format!("{}/{}", - package.manifest().name(), - package.manifest().version())); - add_path_into_database(conn, &prefix, crate_doc_path) - } - - - /// This function will build an empty crate and will add essential documentation files. - /// - /// It is required to run after every rustc update. cratesfyi is not keeping this files - /// for every crate to avoid duplications. - /// - /// List of the files: - /// - /// * rustdoc.css (with rustc version) - /// * main.css (with rustc version) - /// * main.js (with rustc version) - /// * jquery.js (with rustc version) - /// * playpen.js (with rustc version) - /// * normalize.css - /// * FiraSans-Medium.woff - /// * FiraSans-Regular.woff - /// * Heuristica-Italic.woff - /// * SourceCodePro-Regular.woff - /// * SourceCodePro-Semibold.woff - /// * SourceSerifPro-Bold.woff - /// * SourceSerifPro-Regular.woff - pub fn add_essential_files(&self) -> Result<()> { - use std::fs::{copy, create_dir_all}; - - // acme-client-0.0.0 is an empty library crate and it will always build - let pkg = try!(get_package("acme-client", Some("=0.0.0"))); - let res = self.build_package_in_chroot(&pkg, None); - let rustc_version = parse_rustc_version(&res.rustc_version)?; - - if !res.build_success { - return Err(format_err!("Failed to build empty crate for: {}", res.rustc_version)); - } - - info!("Copying essential files for: {}", res.rustc_version); - - let files = (// files require rustc version subfix - ["brush.svg", - "wheel.svg", - "down-arrow.svg", - "dark.css", - "light.css", - "main.js", - "normalize.css", - "rustdoc.css", - "settings.css", - "settings.js", - "storage.js", - "theme.js", - "source-script.js", - "noscript.css", - "rust-logo.png"], - // favicon.ico is not needed because we set our own - // files doesn't require rustc version subfix - ["FiraSans-Medium.woff", - "FiraSans-Regular.woff", - "SourceCodePro-Regular.woff", - "SourceCodePro-Semibold.woff", - "SourceSerifPro-Bold.ttf.woff", - "SourceSerifPro-Regular.ttf.woff", - "SourceSerifPro-It.ttf.woff"]); - - let source = PathBuf::from(&self.options.chroot_path) - .join("home") - .join(&self.options.chroot_user) - .join("cratesfyi") - .join("doc"); - - // use copy_documentation destination directory so self.clean can remove it when - // we are done - let destination = PathBuf::from(&self.options.destination) - .join(format!("{}/{}", pkg.manifest().name(), pkg.manifest().version())); - try!(create_dir_all(&destination)); - - for file in files.0.iter() { - let spl: Vec<&str> = file.split('.').collect(); - let file_name = format!("{}-{}.{}", spl[0], rustc_version, spl[1]); - let source_path = source.join(&file_name); - let destination_path = destination.join(&file_name); - try!(copy(&source_path, &destination_path) - .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display()))); - } - - for file in files.1.iter() { - let source_path = source.join(file); - let destination_path = destination.join(file); - try!(copy(&source_path, &destination_path) - .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display()))); - } - - let conn = try!(connect_db()); - try!(add_path_into_database(&conn, "", destination)); - - try!(self.clean(&pkg)); - - let (vers, _) = self.get_versions(); - - try!(conn.query("INSERT INTO config (name, value) VALUES ('rustc_version', $1)", - &[&vers.to_json()]) - .or_else(|_| { - conn.query("UPDATE config SET value = $1 WHERE name = 'rustc_version'", - &[&vers.to_json()]) - })); - - Ok(()) - } -} - - -/// Returns canonical name of a package. -/// -/// It's just package-version. All directory structure used in cratesfyi is -/// following this naming scheme. -fn canonical_name(package: &Package) -> String { - format!("{}-{}", - package.manifest().name(), - package.manifest().version()) -} - - -/// Runs `func` with the all crates from crates-io.index repository path. -/// -/// First argument of func is the name of crate and -/// second argument is the version of crate. Func will be run for every crate. -fn crates(path: PathBuf, mut func: F) -> Result<()> - where F: FnMut(&str, &str) -> () -{ - crates_from_path(&path, &mut func) -} - -fn mount_for_path(path: &Path) -> Filesystem { - let system = System::new(); - - let mut found = None; - let mut found_pos = std::usize::MAX; - for mount in system.mounts().expect("get all mounts").into_iter() { - let mount_path = Path::new(&mount.fs_mounted_on); - for (i, ancestor) in path.ancestors().enumerate() { - if ancestor == mount_path && i < found_pos { - found_pos = i; - found = Some(mount); - break; - } - } - } - found.expect("on a disk mount") -} - - -#[cfg(test)] -mod test { - extern crate env_logger; - use std::path::PathBuf; - use {DocBuilder, DocBuilderOptions}; - - #[test] - #[ignore] - fn test_build_world() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let mut docbuilder = DocBuilder::new(options); - // This test is building WHOLE WORLD and may take forever - assert!(docbuilder.build_world().is_ok()); - } - - #[test] - #[ignore] - fn test_build_package() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let mut docbuilder = DocBuilder::new(options); - let res = docbuilder.build_package("rand", "0.3.14"); - assert!(res.is_ok()); - } - - #[test] - #[ignore] - fn test_add_essential_files() { - let _ = env_logger::try_init(); - let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix")); - let docbuilder = DocBuilder::new(options); - - docbuilder.add_essential_files().unwrap(); - } -} diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs new file mode 100644 index 000000000..10a28734d --- /dev/null +++ b/src/docbuilder/limits.rs @@ -0,0 +1,92 @@ +use error::Result; +use postgres::Connection; +use std::collections::BTreeMap; +use std::time::Duration; + +pub(crate) struct Limits { + memory: usize, + timeout: Duration, + networking: bool, + max_log_size: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + memory: 3 * 1024 * 1024 * 1024, // 3 GB + timeout: Duration::from_secs(15 * 60), // 15 minutes + networking: false, + max_log_size: 100 * 1024, // 100 KB + } + } +} + +impl Limits { + pub(crate) fn for_crate(conn: &Connection, name: &str) -> Result { + let mut limits = Self::default(); + + let res = conn.query( + "SELECT * FROM sandbox_overrides WHERE crate_name = $1;", + &[&name], + )?; + if !res.is_empty() { + let row = res.get(0); + if let Some(memory) = row.get::<_, Option>("max_memory_bytes") { + limits.memory = memory as usize; + } + if let Some(timeout) = row.get::<_, Option>("timeout_seconds") { + limits.timeout = Duration::from_secs(timeout as u64); + } + } + + Ok(limits) + } + + pub(crate) fn memory(&self) -> usize { + self.memory + } + + pub(crate) fn timeout(&self) -> Duration { + self.timeout + } + + pub(crate) fn networking(&self) -> bool { + self.networking + } + + pub(crate) fn max_log_size(&self) -> usize { + self.max_log_size + } + + pub(crate) fn for_website(&self) -> BTreeMap { + let time_scale = |v| scale(v, 60, &["seconds", "minutes", "hours"]); + let size_scale = |v| scale(v, 1024, &["bytes", "KB", "MB", "GB"]); + + let mut res = BTreeMap::new(); + res.insert("Available RAM".into(), size_scale(self.memory)); + res.insert( + "Maximum rustdoc execution time".into(), + time_scale(self.timeout.as_secs() as usize), + ); + res.insert("Maximum size of a build log".into(), size_scale(self.max_log_size)); + if self.networking { + res.insert("Network access".into(), "allowed".into()); + } else { + res.insert("Network access".into(), "blocked".into()); + } + res + } +} + +fn scale(mut value: usize, interval: usize, labels: &[&str]) -> String { + let mut chosen_label = &labels[0]; + for label in &labels[1..] { + if (value as f64) / (interval as f64) >= 1.0 { + chosen_label = label; + value = value / interval; + } else { + break; + } + } + format!("{} {}", value, chosen_label) +} diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs index e99e10cb4..5fdcd41b9 100644 --- a/src/docbuilder/metadata.rs +++ b/src/docbuilder/metadata.rs @@ -1,6 +1,5 @@ use std::path::Path; -use cargo::core::Package; use toml::Value; use error::Result; use failure::err_msg; @@ -60,10 +59,9 @@ pub struct Metadata { impl Metadata { - pub fn from_package(pkg: &Package) -> Result { - let src_path = pkg.manifest_path().parent().ok_or_else(|| err_msg("Source path not available"))?; + pub(crate) fn from_source_dir(source_dir: &Path) -> Result { for c in ["Cargo.toml.orig", "Cargo.toml"].iter() { - let manifest_path = src_path.clone().join(c); + let manifest_path = source_dir.clone().join(c); if manifest_path.exists() { return Ok(Metadata::from_manifest(manifest_path)); } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 635fe2d55..9087ffe14 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,11 +1,14 @@ pub mod options; pub mod metadata; -mod chroot_builder; +mod limits; +mod rustwide_builder; mod crates; mod queue; -pub use self::chroot_builder::ChrootBuilderResult; +pub use self::rustwide_builder::RustwideBuilder; +pub(crate) use self::rustwide_builder::BuildResult; +pub(crate) use self::limits::Limits; use std::fs; @@ -119,4 +122,15 @@ impl DocBuilder { pub fn options(&self) -> &DocBuilderOptions { &self.options } + + fn add_to_cache(&mut self, name: &str, version: &str) { + self.cache.insert(format!("{}-{}", name, version)); + } + + fn should_build(&self, name: &str, version: &str) -> bool { + let name = format!("{}-{}", name, version); + let local = self.options.skip_if_log_exists && self.cache.contains(&name); + let db = self.options.skip_if_exists && self.db_cache.contains(&name); + !(local || db) + } } diff --git a/src/docbuilder/options.rs b/src/docbuilder/options.rs index 4726acc63..28d5c63b4 100644 --- a/src/docbuilder/options.rs +++ b/src/docbuilder/options.rs @@ -9,9 +9,6 @@ pub struct DocBuilderOptions { pub keep_build_directory: bool, pub prefix: PathBuf, pub destination: PathBuf, - pub chroot_path: PathBuf, - pub chroot_user: String, - pub container_name: String, pub crates_io_index_path: PathBuf, pub skip_if_exists: bool, pub skip_if_log_exists: bool, @@ -27,18 +24,14 @@ impl Default for DocBuilderOptions { let cwd = env::current_dir().unwrap(); - let (prefix, destination, chroot_path, crates_io_index_path) = + let (prefix, destination, crates_io_index_path) = generate_paths(cwd); DocBuilderOptions { prefix: prefix, destination: destination, - chroot_path: chroot_path, crates_io_index_path: crates_io_index_path, - chroot_user: "cratesfyi".to_string(), - container_name: env::var("CRATESFYI_CONTAINER_NAME").expect("CRATESFYI_CONTAINER_NAME"), - keep_build_directory: false, skip_if_exists: false, skip_if_log_exists: false, @@ -53,16 +46,12 @@ impl Default for DocBuilderOptions { impl fmt::Debug for DocBuilderOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, - "DocBuilderOptions {{ destination: {:?}, chroot_path: {:?}, \ + "DocBuilderOptions {{ destination: {:?}, \ crates_io_index_path: {:?}, \ - container_name: {:?}, chroot_user: {:?}, \ keep_build_directory: {:?}, skip_if_exists: {:?}, \ skip_if_log_exists: {:?}, debug: {:?} }}", self.destination, - self.chroot_path, self.crates_io_index_path, - self.container_name, - self.chroot_user, self.keep_build_directory, self.skip_if_exists, self.skip_if_log_exists, @@ -74,12 +63,11 @@ impl fmt::Debug for DocBuilderOptions { impl DocBuilderOptions { /// Creates new DocBuilderOptions from prefix pub fn from_prefix(prefix: PathBuf) -> DocBuilderOptions { - let (prefix, destination, chroot_path, crates_io_index_path) = + let (prefix, destination, crates_io_index_path) = generate_paths(prefix); DocBuilderOptions { prefix: prefix, destination: destination, - chroot_path: chroot_path, crates_io_index_path: crates_io_index_path, ..Default::default() @@ -91,9 +79,6 @@ impl DocBuilderOptions { if !self.destination.exists() { bail!("destination path '{}' does not exist", self.destination.display()); } - if !self.chroot_path.exists() { - bail!("chroot path '{}' does not exist", self.chroot_path.display()); - } if !self.crates_io_index_path.exists() { bail!("crates.io-index path '{}' does not exist", self.crates_io_index_path.display()); } @@ -103,11 +88,9 @@ impl DocBuilderOptions { -fn generate_paths(prefix: PathBuf) -> (PathBuf, PathBuf, PathBuf, PathBuf) { - let name = env::var_os("CRATESFYI_CONTAINER_NAME").expect("CRATESFYI_CONTAINER_NAME"); +fn generate_paths(prefix: PathBuf) -> (PathBuf, PathBuf, PathBuf) { let destination = PathBuf::from(&prefix).join("documentations"); - let chroot_path = PathBuf::from(&prefix).join(&name).join("rootfs"); let crates_io_index_path = PathBuf::from(&prefix).join("crates.io-index"); - (prefix, destination, chroot_path, crates_io_index_path) + (prefix, destination, crates_io_index_path) } diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index ae07e0a5e..f6448b489 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -1,7 +1,6 @@ //! Updates crates.io index and builds new packages - -use super::DocBuilder; +use super::{DocBuilder, RustwideBuilder}; use db::connect_db; use error::Result; use crates_index_diff::{ChangeKind, Index}; @@ -38,7 +37,10 @@ impl DocBuilder { } /// Builds the top package from the queue. Returns whether the queue was empty. - pub fn build_next_queue_package(&mut self) -> Result { + pub(crate) fn build_next_queue_package( + &mut self, + builder: &mut RustwideBuilder, + ) -> Result { let conn = try!(connect_db()); let query = try!(conn.query("SELECT id, name, version @@ -57,7 +59,7 @@ impl DocBuilder { let name: String = query.get(0).get(1); let version: String = query.get(0).get(2); - match self.build_package(&name[..], &version[..]) { + match builder.build_package(self, &name, &version) { Ok(_) => { let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]); } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs new file mode 100644 index 000000000..7218337b0 --- /dev/null +++ b/src/docbuilder/rustwide_builder.rs @@ -0,0 +1,459 @@ +use super::DocBuilder; +use db::file::add_path_into_database; +use db::{add_build_into_database, add_package_into_database, connect_db}; +use docbuilder::{crates::crates_from_path, Limits}; +use error::Result; +use failure::ResultExt; +use log::LevelFilter; +use postgres::Connection; +use rustc_serialize::json::ToJson; +use rustwide::cmd::{Command, SandboxBuilder}; +use rustwide::logging::{self, LogStorage}; +use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; +use std::path::Path; +use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; +use Metadata; + +static USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; +static DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; + +static TARGETS: &[&str] = &[ + "i686-apple-darwin", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", +]; +static DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; + +static ESSENTIAL_FILES_VERSIONED: &[&str] = &[ + "brush.svg", + "wheel.svg", + "down-arrow.svg", + "dark.css", + "light.css", + "main.js", + "normalize.css", + "rustdoc.css", + "settings.css", + "settings.js", + "storage.js", + "theme.js", + "source-script.js", + "noscript.css", + "rust-logo.png", +]; +static ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ + "FiraSans-Medium.woff", + "FiraSans-Regular.woff", + "SourceCodePro-Regular.woff", + "SourceCodePro-Semibold.woff", + "SourceSerifPro-Bold.ttf.woff", + "SourceSerifPro-Regular.ttf.woff", + "SourceSerifPro-It.ttf.woff", +]; + +static DUMMY_CRATE_NAME: &str = "acme-client"; +static DUMMY_CRATE_VERSION: &str = "0.0.0"; + +pub struct RustwideBuilder { + workspace: Workspace, + toolchain: Toolchain, + rustc_version: String, +} + +impl RustwideBuilder { + pub fn init() -> Result { + let env_workspace_path = ::std::env::var("CRATESFYI_RUSTWIDE_WORKSPACE"); + let workspace_path = env_workspace_path + .as_ref() + .map(|v| v.as_str()) + .unwrap_or(DEFAULT_RUSTWIDE_WORKSPACE); + let workspace = WorkspaceBuilder::new(Path::new(workspace_path), USER_AGENT).init()?; + workspace.purge_all_build_dirs()?; + + let toolchain = Toolchain::Dist { + name: "nightly".into(), + }; + + Ok(RustwideBuilder { + workspace, + toolchain, + rustc_version: String::new(), + }) + } + + fn update_toolchain(&mut self) -> Result<()> { + // Ignore errors if detection fails. + let old_version = self.detect_rustc_version().ok(); + + self.toolchain.install(&self.workspace)?; + for target in TARGETS { + self.toolchain.add_target(&self.workspace, target)?; + } + self.rustc_version = self.detect_rustc_version()?; + + if old_version.as_ref().map(|s| s.as_str()) != Some(&self.rustc_version) { + self.add_essential_files()?; + } + + Ok(()) + } + + fn detect_rustc_version(&self) -> Result { + info!("detecting rustc's version..."); + let res = Command::new(&self.workspace, self.toolchain.rustc()) + .args(&["--version"]) + .log_output(false) + .run_capture()?; + let mut iter = res.stdout_lines().iter(); + if let (Some(line), None) = (iter.next(), iter.next()) { + info!("found rustc {}", line); + Ok(line.clone()) + } else { + Err(::failure::err_msg( + "invalid output returned by `rustc --version`", + )) + } + } + + pub fn add_essential_files(&self) -> Result<()> { + info!("building a dummy crate to get essential files"); + let rustc_version = parse_rustc_version(&self.rustc_version)?; + + let conn = connect_db()?; + let limits = Limits::for_crate(&conn, DUMMY_CRATE_NAME)?; + + let mut build_dir = self + .workspace + .build_dir(&format!("essential-files-{}", rustc_version)); + build_dir.purge()?; + + // acme-client-0.0.0 is an empty library crate and it will always build + let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); + krate.fetch(&self.workspace)?; + + let sandbox = SandboxBuilder::new() + .memory_limit(Some(limits.memory())) + .enable_networking(limits.networking()); + + build_dir + .build(&self.toolchain, &krate, sandbox) + .run(|build| { + let res = self.execute_build(None, build, &limits)?; + if !res.successful { + bail!("failed to build dummy crate for {}", self.rustc_version); + } + + info!("copying essential files for {}", self.rustc_version); + let source = build.host_target_dir().join(&res.target).join("doc"); + let dest = ::tempdir::TempDir::new("essential-files")?; + + let files = ESSENTIAL_FILES_VERSIONED + .iter() + .map(|f| (f, true)) + .chain(ESSENTIAL_FILES_UNVERSIONED.iter().map(|f| (f, false))); + for (file, versioned) in files { + let segments = file.rsplitn(2, '.').collect::>(); + let file_name = if versioned { + format!("{}-{}.{}", segments[1], rustc_version, segments[0]) + } else { + file.to_string() + }; + let source_path = source.join(&file_name); + let dest_path = dest.path().join(&file_name); + ::std::fs::copy(&source_path, &dest_path).with_context(|_| { + format!( + "couldn't copy '{}' to '{}'", + source_path.display(), + dest_path.display() + ) + })?; + } + + add_path_into_database(&conn, "", &dest)?; + conn.query( + "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ + ON CONFLICT (name) DO UPDATE SET value = $1;", + &[&self.rustc_version.to_json()], + )?; + + Ok(()) + })?; + + build_dir.purge()?; + krate.purge_from_cache(&self.workspace)?; + Ok(()) + } + + pub fn build_world(&mut self, doc_builder: &mut DocBuilder) -> Result<()> { + let mut count = 0; + crates_from_path( + &doc_builder.options().crates_io_index_path.clone(), + &mut |name, version| { + match self.build_package(doc_builder, name, version) { + Ok(status) => { + count += 1; + if status && count % 10 == 0 { + let _ = doc_builder.save_cache(); + } + } + Err(err) => warn!("failed to build package {} {}: {}", name, version, err), + } + doc_builder.add_to_cache(name, version); + }, + ) + } + + pub fn build_package( + &mut self, + doc_builder: &mut DocBuilder, + name: &str, + version: &str, + ) -> Result { + if !doc_builder.should_build(name, version) { + return Ok(false); + } + + self.update_toolchain()?; + + info!("building package {} {}", name, version); + + let conn = connect_db()?; + let limits = Limits::for_crate(&conn, name)?; + + let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version)); + build_dir.purge()?; + + let krate = Crate::crates_io(name, version); + krate.fetch(&self.workspace)?; + + let sandbox = SandboxBuilder::new() + .memory_limit(Some(limits.memory())) + .enable_networking(limits.networking()); + + let res = build_dir + .build(&self.toolchain, &krate, sandbox) + .run(|build| { + let mut files_list = None; + let mut has_docs = false; + let mut successful_targets = Vec::new(); + + // Do an initial build and then copy the sources in the database + let res = self.execute_build(None, &build, &limits)?; + if res.successful { + debug!("adding sources into database"); + let prefix = format!("sources/{}/{}", name, version); + files_list = Some(add_path_into_database( + &conn, + &prefix, + build.host_source_dir(), + )?); + + has_docs = build + .host_target_dir() + .join(&res.target) + .join("doc") + .join(name.replace("-", "_")) + .is_dir(); + } + + if has_docs { + debug!("adding documentation for the default target to the database"); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + &res.target, + true, + )?; + + // Then build the documentation for all the targets + for target in TARGETS { + debug!("building package {} {} for {}", name, version, target); + let target_res = self.execute_build(Some(target), &build, &limits)?; + if target_res.successful { + // Cargo is not giving any error and not generating documentation of some crates + // when we use a target compile options. Check documentation exists before + // adding target to successfully_targets. + if build.host_target_dir().join(target).join("doc").is_dir() { + debug!( + "adding documentation for target {} to the database", + target + ); + self.copy_docs( + &doc_builder, + &build.host_target_dir(), + name, + version, + target, + false, + )?; + successful_targets.push(target.to_string()); + } + } + } + + self.upload_docs(doc_builder, &conn, name, version)?; + } + + let has_examples = build.host_source_dir().join("examples").is_dir(); + let release_id = add_package_into_database( + &conn, + res.cargo_metadata.root(), + &build.host_source_dir(), + &res, + files_list, + successful_targets, + has_docs, + has_examples, + )?; + add_build_into_database(&conn, &release_id, &res)?; + + doc_builder.add_to_cache(name, version); + Ok(res) + })?; + + build_dir.purge()?; + krate.purge_from_cache(&self.workspace)?; + Ok(res.successful) + } + + fn execute_build( + &self, + target: Option<&str>, + build: &Build, + limits: &Limits, + ) -> Result { + let metadata = Metadata::from_source_dir(&build.host_source_dir())?; + let cargo_metadata = + CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?; + + let target = if let Some(target) = target { + target + } else if let Some(target) = metadata.default_target.as_ref().map(|s| s.as_str()) { + target + } else { + DEFAULT_TARGET + } + .to_string(); + + let mut rustdoc_flags: Vec = vec![ + "-Z".to_string(), + "unstable-options".to_string(), + "--resource-suffix".to_string(), + format!("-{}", parse_rustc_version(&self.rustc_version)?), + "--static-root-path".to_string(), + "/".to_string(), + "--disable-per-crate-search".to_string(), + ]; + for dep in &cargo_metadata.root_dependencies() { + rustdoc_flags.push("--extern-html-root-url".to_string()); + rustdoc_flags.push(format!( + "{}=https://docs.rs/{}/{}", + dep.name.replace("-", "_"), + dep.name, + dep.version + )); + } + let mut cargo_args = vec![ + "doc".to_owned(), + "--lib".to_owned(), + "--no-deps".to_owned(), + "--target".to_owned(), + target.to_owned(), + ]; + if let Some(features) = &metadata.features { + cargo_args.push("--features".to_owned()); + cargo_args.push(features.join(" ")); + } + if metadata.all_features { + cargo_args.push("--all-features".to_owned()); + } + if metadata.no_default_features { + cargo_args.push("--no-default-features".to_owned()); + } + + let mut storage = LogStorage::new(LevelFilter::Info); + storage.set_max_size(limits.max_log_size()); + + let successful = logging::capture(&storage, || { + build + .cargo() + .timeout(Some(limits.timeout())) + .no_output_timeout(None) + .env( + "RUSTFLAGS", + metadata + .rustc_args + .map(|args| args.join("")) + .unwrap_or("".to_owned()), + ) + .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) + .args(&cargo_args) + .run() + .is_ok() + }); + + Ok(BuildResult { + build_log: storage.to_string(), + rustc_version: self.rustc_version.clone(), + docsrs_version: format!("docsrs {}", ::BUILD_VERSION), + successful, + cargo_metadata, + target: target.to_string(), + }) + } + + fn copy_docs( + &self, + doc_builder: &DocBuilder, + target_dir: &Path, + name: &str, + version: &str, + target: &str, + is_default_target: bool, + ) -> Result<()> { + let source = target_dir.join(target); + + let mut dest = doc_builder.options().destination.join(name).join(version); + // only add target name to destination directory when we are copying a non-default target. + // this is allowing us to host documents in the root of the crate documentation directory. + // for example winapi will be available in docs.rs/winapi/$version/winapi/ for it's + // default target: x86_64-pc-windows-msvc. But since it will be built under + // cratesfyi/x86_64-pc-windows-msvc we still need target in this function. + if !is_default_target { + dest = dest.join(target); + } + + info!("{} {}", source.display(), dest.display()); + copy_doc_dir(source, dest, self.rustc_version.trim())?; + Ok(()) + } + + fn upload_docs( + &self, + doc_builder: &DocBuilder, + conn: &Connection, + name: &str, + version: &str, + ) -> Result<()> { + debug!("Adding documentation into database"); + let prefix = format!("rustdoc/{}/{}", name, version); + let database_prefix = + Path::new(&doc_builder.options().destination).join(format!("{}/{}", name, version)); + add_path_into_database(conn, &prefix, database_prefix)?; + Ok(()) + } +} + +pub(crate) struct BuildResult { + pub(crate) rustc_version: String, + pub(crate) docsrs_version: String, + pub(crate) build_log: String, + pub(crate) successful: bool, + target: String, + cargo_metadata: CargoMetadata, +} diff --git a/src/lib.rs b/src/lib.rs index 0da58ca8c..4ab8cdaf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,9 +40,11 @@ extern crate rusoto_credential; extern crate futures; extern crate tokio; extern crate systemstat; +extern crate rustwide; +extern crate tempdir; +pub use self::docbuilder::RustwideBuilder; pub use self::docbuilder::DocBuilder; -pub use self::docbuilder::ChrootBuilderResult; pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::metadata::Metadata; pub use self::web::start_web_server; diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs deleted file mode 100644 index 7fb9d69b5..000000000 --- a/src/utils/build_doc.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Crate documentation builder -//! -//! This module is extremely similar to cargo install operation, except it's building -//! documentation of a crate and not installing anything. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::env; -use std::sync::Arc; - -use cargo::core::{self, SourceId, Dependency, Source, Package, Workspace}; -use cargo::core::compiler::{DefaultExecutor, CompileMode, MessageFormat, BuildConfig, Executor}; -use cargo::core::package::PackageSet; -use cargo::core::registry::PackageRegistry; -use cargo::core::resolver; -use cargo::core::source::SourceMap; -use cargo::util::{CargoResult, Config, internal, Filesystem}; -use cargo::sources::SourceConfigMap; -use cargo::ops::{self, Packages, LibRule, FilterRule}; - -use utils::{get_current_versions, parse_rustc_version}; -use error::Result; - -use Metadata; - - -/// Builds documentation of a crate and version. -/// -/// Crate will be built into current working directory/crate-version. -/// -/// It will build latest version, if no version is given. -// idea is to make cargo to download -// and build a crate and its documentation -// instead of doing it manually like in the previous version of cratesfyi -pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> Result { - core::enable_nightly_features(); - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let source_cfg_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_cfg_map.load(source_id, &HashSet::new())); - - let _lock = try!(config.acquire_package_cache_lock()); - - // update crates.io-index registry - try!(source.update()); - - let dep = try!(Dependency::parse_no_deprecated(name, vers, source_id)); - let deps = try!(source.query_vec(&dep)); - let pkgid = try!(deps.iter().map(|p| p.package_id()).max() - // FIXME: This is probably not a rusty way to handle options and results - // or maybe it is who knows... - .ok_or(internal("no package id available"))); - - let mut source_map = SourceMap::new(); - source_map.insert(source); - - let pkg_set = try!(PackageSet::new(&[pkgid.clone()], source_map, &config)); - - let pkg = try!(pkg_set.get_one(pkgid)).clone(); - - let current_dir = try!(env::current_dir()); - let target_dir = PathBuf::from(current_dir).join("cratesfyi"); - - let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?; - - // This is only way to pass rustc_args to cargo. - // CompileOptions::target_rustc_args is used only for the current crate, - // and since docs.rs never runs rustc on the current crate, we assume rustc_args - // will be used for the dependencies. That is why we are creating RUSTFLAGS environment - // variable instead of using target_rustc_args. - if let Some(rustc_args) = metadata.rustc_args { - env::set_var("RUSTFLAGS", rustc_args.join(" ")); - } - - // since https://github.com/rust-lang/rust/pull/48511 we can pass --resource-suffix to - // add correct version numbers to css and javascript files - let mut rustdoc_args: Vec = - vec!["-Z".to_string(), "unstable-options".to_string(), - "--resource-suffix".to_string(), - format!("-{}", parse_rustc_version(get_current_versions()?.0)?), - "--static-root-path".to_string(), "/".to_string(), - "--disable-per-crate-search".to_string()]; - - // since https://github.com/rust-lang/rust/pull/51384, we can pass --extern-html-root-url to - // force rustdoc to link to other docs.rs docs for dependencies - let source = try!(source_cfg_map.load(source_id, &HashSet::new())); - for (name, dep) in try!(resolve_deps(&pkg, &config, source)) { - rustdoc_args.push("--extern-html-root-url".to_string()); - rustdoc_args.push(format!("{}=https://docs.rs/{}/{}", - name.replace("-", "_"), dep.name(), dep.version())); - } - - if let Some(package_rustdoc_args) = metadata.rustdoc_args { - rustdoc_args.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect()); - } - - let mut build_config = try!(BuildConfig::new(&config, - None, - &target.map(|t| t.to_string()), - CompileMode::Doc { deps: false })); - build_config.release = false; - build_config.message_format = MessageFormat::Human; - - let opts = ops::CompileOptions { - config: &config, - build_config, - features: metadata.features.unwrap_or(Vec::new()), - all_features: metadata.all_features, - no_default_features: metadata.no_default_features, - spec: Packages::Packages(Vec::new()), - filter: ops::CompileFilter::new(LibRule::True, - FilterRule::none(), - FilterRule::none(), - FilterRule::none(), - FilterRule::none()), - target_rustc_args: None, - target_rustdoc_args: Some(rustdoc_args), - local_rustdoc_args: None, - export_dir: None, - }; - - let ws = try!(Workspace::ephemeral(pkg, &config, Some(Filesystem::new(target_dir)), false)); - let exec: Arc = Arc::new(DefaultExecutor); - try!(ops::compile_ws(&ws, &opts, &exec)); - - Ok(try!(ws.current()).clone()) -} - -fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box) - -> CargoResult> -{ - let mut registry = try!(PackageRegistry::new(config)); - registry.add_preloaded(src); - registry.lock_patches(); - - let resolver = try!(resolver::resolve( - &[(pkg.summary().clone(), resolver::Method::Everything)], - pkg.manifest().replace(), - &mut registry, - &Default::default(), - None, - false, - )); - let dep_ids = resolver.deps(pkg.package_id()).map(|p| p.0).collect::>(); - let pkg_set = try!(registry.get(&dep_ids)); - let deps = try!(pkg_set.get_many(dep_ids)); - - let mut ret = Vec::new(); - for dep in deps { - if let Some(d) = pkg.dependencies().iter().find(|d| d.package_name() == dep.name()) { - ret.push((d.name_in_toml().to_string(), dep.clone())); - } - } - - Ok(ret) -} - -/// Downloads a crate and returns Cargo Package. -pub fn get_package(name: &str, vers: Option<&str>) -> CargoResult { - core::enable_nightly_features(); - debug!("Getting package with cargo"); - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let source_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_map.load(source_id, &HashSet::new())); - - let _lock = try!(config.acquire_package_cache_lock()); - - try!(source.update()); - - let dep = try!(Dependency::parse_no_deprecated(name, vers, source_id)); - let deps = try!(source.query_vec(&dep)); - let pkgid = try!(deps.iter().map(|p| p.package_id()).max() - // FIXME: This is probably not a rusty way to handle options and results - // or maybe it is who knows... - .ok_or(internal("no package id available"))); - - let mut source_map = SourceMap::new(); - source_map.insert(source); - - let pkg_set = try!(PackageSet::new(&[pkgid.clone()], source_map, &config)); - - let pkg = try!(pkg_set.get_one(pkgid)).clone(); - - Ok(pkg) -} - - -/// Updates central crates-io.index repository -pub fn update_sources() -> CargoResult<()> { - let config = try!(Config::default()); - let source_id = try!(SourceId::crates_io(&config)); - - let _lock = try!(config.acquire_package_cache_lock()); - - let source_map = try!(SourceConfigMap::new(&config)); - let mut source = try!(source_map.load(source_id, &HashSet::new())); - - source.update() -} - - -/// Gets source path of a downloaded package. -pub fn source_path(pkg: &Package) -> Option<&Path> { - // parent of the manifest file is where source codes are stored - pkg.manifest_path().parent() -} - - - - -#[cfg(test)] -mod test { - use std::path::Path; - use super::*; - - #[test] - fn test_get_package() { - let pkg = get_package("rand", None); - assert!(pkg.is_ok()); - - let pkg = pkg.unwrap(); - - let manifest = pkg.manifest(); - assert_eq!(manifest.name().as_str(), "rand"); - } - - - #[test] - fn test_source_path() { - let pkg = get_package("rand", None).unwrap(); - let source_path = source_path(&pkg).unwrap(); - assert!(source_path.is_dir()); - - let cargo_toml_path = Path::new(source_path).join("Cargo.toml"); - assert!(cargo_toml_path.exists()); - assert!(cargo_toml_path.is_file()); - - let src_path = Path::new(source_path).join("src"); - assert!(src_path.exists()); - assert!(src_path.is_dir()); - } -} diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs new file mode 100644 index 000000000..79f09bb48 --- /dev/null +++ b/src/utils/cargo_metadata.rs @@ -0,0 +1,116 @@ +use error::Result; +use rustwide::{cmd::Command, Toolchain, Workspace}; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +pub(crate) struct CargoMetadata { + packages: HashMap, + deps_graph: HashMap>, + root_id: String, +} + +impl CargoMetadata { + pub(crate) fn load( + workspace: &Workspace, + toolchain: &Toolchain, + source_dir: &Path, + ) -> Result { + let res = Command::new(workspace, toolchain.cargo()) + .args(&["metadata", "--format-version", "1"]) + .cd(source_dir) + .log_output(false) + .run_capture()?; + + let mut iter = res.stdout_lines().iter(); + let metadata = if let (Some(serialized), None) = (iter.next(), iter.next()) { + ::rustc_serialize::json::decode::(serialized)? + } else { + return Err(::failure::err_msg( + "invalid output returned by `cargo metadata`", + )); + }; + + // Convert from Vecs to HashMaps and HashSets to get more efficient lookups + Ok(CargoMetadata { + packages: metadata + .packages + .into_iter() + .map(|pkg| (pkg.id.clone(), pkg)) + .collect(), + deps_graph: metadata + .resolve + .nodes + .into_iter() + .map(|node| (node.id, node.deps.into_iter().map(|d| d.pkg).collect())) + .collect(), + root_id: metadata.resolve.root, + }) + } + + pub(crate) fn root_dependencies(&self) -> Vec<&Package> { + let ids = &self.deps_graph[&self.root_id]; + self.packages + .iter() + .filter(|(id, _pkg)| ids.contains(id.as_str())) + .map(|(_id, pkg)| pkg) + .collect() + } + + pub(crate) fn root(&self) -> &Package { + &self.packages[&self.root_id] + } +} + +#[derive(RustcDecodable)] +pub(crate) struct Package { + pub(crate) id: String, + pub(crate) name: String, + pub(crate) version: String, + pub(crate) license: Option, + pub(crate) repository: Option, + pub(crate) homepage: Option, + pub(crate) description: Option, + pub(crate) documentation: Option, + pub(crate) dependencies: Vec, + pub(crate) targets: Vec, + pub(crate) readme: Option, + pub(crate) keywords: Vec, + pub(crate) authors: Vec, +} + +#[derive(RustcDecodable)] +pub(crate) struct Target { + pub(crate) name: String, + pub(crate) kind: Vec, + pub(crate) src_path: Option, +} + +#[derive(RustcDecodable)] +pub(crate) struct Dependency { + pub(crate) name: String, + pub(crate) req: String, + pub(crate) kind: Option, +} + +#[derive(RustcDecodable)] +struct DeserializedMetadata { + packages: Vec, + resolve: DeserializedResolve, +} + +#[derive(RustcDecodable)] +struct DeserializedResolve { + root: String, + nodes: Vec, +} + +#[derive(RustcDecodable)] +struct DeserializedResolveNode { + id: String, + deps: Vec, +} + +#[derive(RustcDecodable)] +struct DeserializedResolveDep { + pkg: String, +} diff --git a/src/utils/copy.rs b/src/utils/copy.rs index bc73c8dd1..28847dbc1 100644 --- a/src/utils/copy.rs +++ b/src/utils/copy.rs @@ -84,7 +84,6 @@ fn copy_files_and_handle_html(source: PathBuf, #[cfg(test)] mod test { extern crate env_logger; - extern crate tempdir; use std::fs; use std::path::Path; use super::*; @@ -101,24 +100,4 @@ mod test { assert!(res.is_ok()); } - - - #[test] - #[ignore] - fn test_copy_doc_dir() { - // lets build documentation of rand crate - use utils::build_doc; - let pkg = build_doc("rand", None, None).unwrap(); - - let pkg_dir = format!("rand-{}", pkg.manifest().version()); - let target = Path::new(&pkg_dir); - let destination = tempdir::TempDir::new("cratesfyi").unwrap(); - let res = copy_doc_dir(target, destination.path(), "UNKNOWN"); - - // remove build and temp dir - fs::remove_dir_all(target).unwrap(); - fs::remove_dir_all(destination.path()).unwrap(); - - assert!(res.is_ok()); - } } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index d8c40677c..59e594d70 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -12,9 +12,10 @@ use std::time::Duration; use std::path::PathBuf; use libc::fork; use time; +use docbuilder::RustwideBuilder; use DocBuilderOptions; use DocBuilder; -use utils::{update_sources, update_release_activity, github_updater, pubsubhubbub}; +use utils::{update_release_activity, github_updater, pubsubhubbub}; use db::{connect_db, update_search_index}; @@ -26,7 +27,7 @@ pub fn start_daemon() { "CRATESFYI_GITHUB_USERNAME", "CRATESFYI_GITHUB_ACCESSTOKEN"] .iter() { - env::var(v).expect("Environment variable not found"); + env::var(v).expect(&format!("Environment variable {} not found", v)); } let dbopts = opts(); @@ -85,6 +86,8 @@ pub fn start_daemon() { QueueInProgress(usize), } + let mut builder = RustwideBuilder::init().unwrap(); + let mut status = BuilderState::Fresh; loop { @@ -116,11 +119,6 @@ pub fn start_daemon() { if let Err(e) = doc_builder.save_cache() { error!("Failed to save cache: {}", e); } - - if let Err(e) = update_sources() { - error!("Failed to update sources: {}", e); - continue; - } } // Only build crates if there are any to build @@ -158,18 +156,13 @@ pub fn start_daemon() { error!("Failed to load cache: {}", e); continue; } - - if let Err(e) = update_sources() { - error!("Failed to update sources: {}", e); - continue; - } } // Run build_packages_queue under `catch_unwind` to catch panics // This only panicked twice in the last 6 months but its just a better // idea to do this. let res = catch_unwind(AssertUnwindSafe(|| { - match doc_builder.build_next_queue_package() { + match doc_builder.build_next_queue_package(&mut builder) { Err(e) => error!("Failed to build crate from queue: {}", e), Ok(crate_built) => if crate_built { status.increment(); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 6f03bc9ac..8db86bd13 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,7 +1,6 @@ //! Various utilities for cratesfyi -pub use self::build_doc::{build_doc, get_package, source_path, update_sources}; pub use self::copy::{copy_dir, copy_doc_dir}; pub use self::github_updater::github_updater; pub use self::release_activity_updater::update_release_activity; @@ -9,9 +8,10 @@ pub use self::daemon::start_daemon; pub use self::rustc_version::{parse_rustc_version, get_current_versions, command_result}; pub use self::html::extract_head_and_body; pub use self::queue::add_crate_to_queue; +pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage}; +mod cargo_metadata; mod github_updater; -mod build_doc; mod copy; mod release_activity_updater; mod daemon; diff --git a/src/web/builds.rs b/src/web/builds.rs index a0eb05530..a8fed29ed 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -1,5 +1,6 @@ +use docbuilder::Limits; use std::collections::BTreeMap; use super::MetaData; use super::pool::Pool; @@ -27,6 +28,7 @@ struct BuildsPage { metadata: Option, builds: Vec, build_details: Option, + limits: Limits, } @@ -54,6 +56,7 @@ impl ToJson for BuildsPage { m.insert("metadata".to_owned(), self.metadata.to_json()); m.insert("builds".to_owned(), self.builds.to_json()); m.insert("build_details".to_owned(), self.build_details.to_json()); + m.insert("limits".into(), self.limits.for_website().to_json()); m.to_json() } } @@ -67,6 +70,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult { let req_build_id: i32 = router.find("id").unwrap_or("0").parse().unwrap_or(0); let conn = extension!(req, Pool); + let limits = ctry!(Limits::for_crate(&conn, name)); let mut build_list: Vec = Vec::new(); let mut build_details = None; @@ -131,6 +135,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult { metadata: MetaData::from_crate(&conn, &name, &version), builds: build_list, build_details: build_details, + limits, }; Page::new(builds_page) .set_true("show_package_navigation") diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index 4bab4fadc..5c8b93ae5 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use iron::prelude::*; use iron::headers::ContentType; -use rustc_serialize::json::Json; +use rustc_serialize::json::{Json, ToJson}; use super::page::Page; use super::pool::Pool; use time; @@ -40,10 +40,12 @@ pub fn about_handler(req: &mut Request) -> IronResult { if let Some(row) = res.iter().next() { if let Some(Ok::(res)) = row.get_opt(0) { if let Some(vers) = res.as_string() { - content.insert("rustc_version".to_string(), vers.to_string()); + content.insert("rustc_version".to_string(), vers.to_json()); } } } + content.insert("limits".to_string(), ::docbuilder::Limits::default().for_website().to_json()); + Page::new(content).title("About Docs.rs").to_resp("about") } diff --git a/templates/about.hbs b/templates/about.hbs index f602ff403..470ea67f2 100644 --- a/templates/about.hbs +++ b/templates/about.hbs @@ -31,6 +31,30 @@ no README will be displayed.

+

Global sandbox limits

+ +

+ All the builds on docs.rs are executed inside a sandbox with limited + resources. The current limits are the following: +

+ + + + {{#each content.limits}} + + + + + {{/each}} + +
{{{@key}}}{{this}}
+ +

+ If a build fails because it hit one of those limits please + open an issue + to get them increased for your crate. +

+

Redirections

diff --git a/templates/builds.hbs b/templates/builds.hbs index c32fac506..630661ea7 100644 --- a/templates/builds.hbs +++ b/templates/builds.hbs @@ -8,14 +8,12 @@

Build #{{build_details.id}} {{build_details.build_time}}
- 
-$ rustc --version
+    
$ rustc --version
 {{build_details.rustc_version}}
 $ cratesfyi --version
 {{build_details.cratesfyi_version}}
 $ cratesfyi ...
-{{build_details.output}}
-    

+{{build_details.output}}
{{/if}}
@@ -36,6 +34,31 @@ $ cratesfyi ... {{/each}} + +
+

{{metadata.name}}'s sandbox limits

+ +

+ All the builds on docs.rs are executed inside a sandbox with limited + resources. The limits for this crate are the following: +

+ + + + {{#each limits}} + + + + + {{/each}} + +
{{{@key}}}{{this}}
+

+ If a build fails because it hit one of those limits please + open an issue + to get them increased. +

+
{{/with}}