Merge branch 'stable-1' of https://git.proxmox.com/git/proxmox-backup

Author: wofferl

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "1.1.10"
+version = "1.1.13"
 authors = [
     "Dietmar Maurer <[email protected]>",
     "Dominik Csapak <[email protected]>",
@@ -52,17 +52,6 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pin-project = "1.0"
-#pathpatterns = "0.1.2"
-pathpatterns = { path = "../pathpatterns" }
-#proxmox = { version = "0.11.5", features = [ "sortable-macro", "api-macro" ] }
-#proxmox = { git = "git://git.proxmox.com/git/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
-proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro" ] }
-#proxmox-fuse = "0.1.1"
-proxmox-fuse = { path = "../proxmox-fuse" }
-#proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] }
-proxmox-http = { version = "0.2.1", path = "../proxmox/proxmox-http", features = [ "client", "http-helpers", "websocket" ] }
-#pxar = { version = "0.10.1", features = [ "tokio-io" ] }
-pxar = { path = "../pxar", features = [ "tokio-io" ] }
 regex = "1.2"
 rustyline = "7"
 serde = { version = "1.0", features = ["derive"] }
@@ -84,8 +73,23 @@ zstd = { version = "0.4", features = [ "bindgen" ] }
 nom = "5.1"
 crossbeam-channel = "0.5"
 
-#proxmox-acme-rs = "0.2.1"
+pathpatterns = "0.1.2"
+pxar = { version = "0.10.1", features = [ "tokio-io" ] }
+
+proxmox = { version = "0.11.6", features = [ "sortable-macro", "api-macro", "cli", "router", "tfa" ] }
+proxmox-acme-rs = "0.2.1"
+proxmox-fuse = "0.1.1"
+proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] }
+
+# Local path overrides
+# NOTE: You must run `cargo update` after changing this for it to take effect!
+[patch.crates-io]
+proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "cli", "router", "tfa" ] }
+proxmox-http = { path = "../proxmox/proxmox-http", features = [ "client", "http-helpers", "websocket" ] }
+proxmox-fuse = { path = "../proxmox-fuse" }
 proxmox-acme-rs = { path = "../proxmox-acme-rs" }
+pxar = { path = "../pxar", features = [ "tokio-io" ] }
+pathpatterns = { path = "../pathpatterns" }
 
 [features]
 default = []

Makefile

@@ -113,7 +113,9 @@ deb: build
 	lintian $(DEBS)
 
 .PHONY: deb-all
-deb-all: $(DOC_DEB) $(DEBS)
+deb-all: build
+	cd build; dpkg-buildpackage -b -us -uc --no-pre-clean
+	lintian $(DEBS) $(DOC_DEB)
 
 .PHONY: dsc
 dsc: $(DSC)

build.rs

@@ -2,23 +2,22 @@
 use std::env;
 use std::process::Command;
 
+fn git_command(args: &[&str]) -> String {
+    match Command::new("git").args(args).output() {
+        Ok(output) => String::from_utf8(output.stdout).unwrap().trim_end().to_string(),
+        Err(err) => {
+            panic!("git {:?} failed: {}", args, err);
+        }
+    }
+}
+
 fn main() {
+    let repo_path = git_command(&["rev-parse", "--show-toplevel"]);
     let repoid = match env::var("REPOID") {
         Ok(repoid) => repoid,
-        Err(_) => {
-            match Command::new("git")
-                .args(&["rev-parse", "HEAD"])
-                .output()
-            {
-                Ok(output) => {
-                    String::from_utf8(output.stdout).unwrap()
-                }
-                Err(err) => {
-                    panic!("git rev-parse failed: {}", err);
-                }
-            }
-        }
+        Err(_) => git_command(&["rev-parse", "HEAD"]),
     };
 
     println!("cargo:rustc-env=REPOID={}", repoid);
+    println!("cargo:rerun-if-changed={}/.git/HEAD", repo_path);
 }

debian/changelog

@@ -1,3 +1,54 @@
+rust-proxmox-backup (1.1.13-2) buster; urgency=medium
+
+  * revert "auth: improve thread safety of 'crypt' C-library", not safe for
+    Debian buster based releases.
+
+ -- Proxmox Support Team <[email protected]>  Mon, 26 Jul 2021 16:40:07 +0200
+
+rust-proxmox-backup (1.1.13-1) buster; urgency=medium
+
+  * auth: improve thread safety of 'crypt' C-library
+
+  * file-restore: increase lock timeout on QEMU map
+
+  * file restore daemon: log basic startup steps
+
+  * REST-API: set error message extension for bad-request response log to
+    ensure the actual error is logged in any (access) log, making debugging
+    such issues easier.
+
+  * restore daemon: use millisecond log resolution
+
+  * fix #3496: acme: plugin: actually sleep after setting the TXT record,
+    ensuring DNS propagation of that record. This makes it catch up with the
+    docs/web-interface, where the option was already available.
+
+ -- Proxmox Support Team <[email protected]>  Fri, 23 Jul 2021 12:34:29 +0200
+
+rust-proxmox-backup (1.1.12-1) buster; urgency=medium
+
+  * subscription: set higher-level error to message instead of bailing out, to
+    ensure a force-check gets through
+
+  * ui: dashboard: datastore stats: fix closing <i> tag
+
+  * ui: datastore: option view: only navigate up when we actually removed the
+    datastore
+
+ -- Proxmox Support Team <[email protected]>  Fri, 09 Jul 2021 12:56:35 +0200
+
+rust-proxmox-backup (1.1.11-1) buster; urgency=medium
+
+  * tape/drive: fix logging when requesting media
+
+  * tape: fix LTO locate_file for HP drives
+
+  * fix #3393 (again): pxar/create: try to read xattrs/fcaps/acls by default
+
+  * proxmox-backup-manager: show task log on datastore create
+
+ -- Proxmox Support Team <[email protected]>  Wed, 30 Jun 2021 11:24:20 +0200
+
 rust-proxmox-backup (1.1.10-1) buster; urgency=medium
 
   * ui: datastore list summary: catch and show errors per datastore

docs/Makefile

@@ -228,6 +228,7 @@ epub3: ${GENERATED_SYNOPSIS}
 
 clean:
 	rm -r -f *~ *.1 ${BUILDDIR} ${GENERATED_SYNOPSIS} api-viewer/apidata.js
+	rm -f api-viewer/apidoc.js lto-barcode/lto-barcode-generator.js
 
 
 install_manual_pages: ${MAN1_PAGES} ${MAN5_PAGES}

src/acme/plugin.rs

@@ -2,6 +2,7 @@ use std::future::Future;
 use std::pin::Pin;
 use std::process::Stdio;
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::{bail, format_err, Error};
 use hyper::{Body, Request, Response};
@@ -68,7 +69,7 @@ fn extract_challenge<'a>(
         .challenges
         .iter()
         .find(|ch| ch.ty == ty)
-        .ok_or_else(|| format_err!("no supported challenge type (dns-01) found"))
+        .ok_or_else(|| format_err!("no supported challenge type ({}) found", ty))
 }
 
 async fn pipe_to_tasklog<T: AsyncRead + Unpin>(
@@ -180,7 +181,21 @@ impl AcmePlugin for DnsPlugin {
         domain: &'d AcmeDomain,
         task: Arc<WorkerTask>,
     ) -> Pin<Box<dyn Future<Output = Result<&'c str, Error>> + Send + 'fut>> {
-        Box::pin(self.action(client, authorization, domain, task, "setup"))
+        Box::pin(async move {
+            let result = self
+                .action(client, authorization, domain, task.clone(), "setup")
+                .await;
+            let validation_delay = self.core.validation_delay.unwrap_or(30) as u64;
+
+            if validation_delay > 0 {
+                task.log(format!(
+                    "Sleeping {} seconds to wait for TXT record propagation",
+                    validation_delay
+                ));
+                tokio::time::sleep(Duration::from_secs(validation_delay)).await;
+            }
+            result
+        })
     }
 
     fn teardown<'fut, 'a: 'fut, 'b: 'fut, 'c: 'fut, 'd: 'fut>(

src/api2/node/disks.rs

@@ -66,6 +66,8 @@ pub fn list_disks(
         }
     }
 
+    list.sort_by(|a, b| a.name.cmp(&b.name));
+
     Ok(list)
 }
 

src/bin/proxmox-restore-daemon.rs

@@ -1,7 +1,7 @@
 ///! Daemon binary to run inside a micro-VM for secure single file restore of disk images
 use anyhow::{bail, format_err, Error};
-use log::error;
 use lazy_static::lazy_static;
+use log::{info, error};
 
 use std::os::unix::{
     io::{FromRawFd, RawFd},
@@ -37,24 +37,31 @@ lazy_static! {
 /// This is expected to be run by 'proxmox-file-restore' within a mini-VM
 fn main() -> Result<(), Error> {
     if !Path::new(VM_DETECT_FILE).exists() {
-        bail!(concat!(
-            "This binary is not supposed to be run manually. ",
-            "Please use 'proxmox-file-restore' instead."
-        ));
+        bail!(
+            "This binary is not supposed to be run manually, use 'proxmox-file-restore' instead."
+        );
     }
 
     // don't have a real syslog (and no persistance), so use env_logger to print to a log file (via
     // stdout to a serial terminal attached by QEMU)
     env_logger::from_env(env_logger::Env::default().default_filter_or("info"))
         .write_style(env_logger::WriteStyle::Never)
+        .format_timestamp_millis()
         .init();
 
+    // the API may save some stuff there, e.g., the memcon tracking file
+    // we do not care much, but it's way less headache to just create it
+    std::fs::create_dir_all("/run/proxmox-backup")?;
+
     // scan all attached disks now, before starting the API
     // this will panic and stop the VM if anything goes wrong
+    info!("scanning all disks...");
     {
         let _disk_state = DISK_STATE.lock().unwrap();
     }
 
+    info!("disk scan complete, starting main runtime...");
+
     proxmox_backup::tools::runtime::main(run())
 }
 

src/bin/proxmox_backup_manager/datastore.rs

@@ -5,6 +5,11 @@ use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
 
 use proxmox_backup::config;
 use proxmox_backup::api2::{self, types::* };
+use proxmox_backup::client::{
+    connect_to_localhost,
+    view_task_result,
+};
+use proxmox_backup::config::datastore::DIR_NAME_SCHEMA;
 
 #[api(
     input: {
@@ -67,6 +72,81 @@ fn show_datastore(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value
     Ok(Value::Null)
 }
 
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            name: {
+                schema: DATASTORE_SCHEMA,
+            },
+            path: {
+                schema: DIR_NAME_SCHEMA,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
+            "gc-schedule": {
+                optional: true,
+                schema: GC_SCHEDULE_SCHEMA,
+            },
+            "prune-schedule": {
+                optional: true,
+                schema: PRUNE_SCHEDULE_SCHEMA,
+            },
+            "keep-last": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_LAST,
+            },
+            "keep-hourly": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_HOURLY,
+            },
+            "keep-daily": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_DAILY,
+            },
+            "keep-weekly": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_WEEKLY,
+            },
+            "keep-monthly": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_MONTHLY,
+            },
+            "keep-yearly": {
+                optional: true,
+                schema: PRUNE_SCHEMA_KEEP_YEARLY,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        },
+    },
+)]
+/// Create new datastore config.
+async fn create_datastore(mut param: Value) -> Result<Value, Error> {
+
+    let output_format = extract_output_format(&mut param);
+
+    let mut client = connect_to_localhost()?;
+
+    let result = client.post(&"api2/json/config/datastore", Some(param)).await?;
+
+    view_task_result(&mut client, result, &output_format).await?;
+
+    Ok(Value::Null)
+}
+
 pub fn datastore_commands() -> CommandLineInterface {
 
     let cmd_def = CliCommandMap::new()
@@ -77,7 +157,7 @@ pub fn datastore_commands() -> CommandLineInterface {
                 .completion_cb("name", config::datastore::complete_datastore_name)
         )
         .insert("create",
-                CliCommand::new(&api2::config::datastore::API_METHOD_CREATE_DATASTORE)
+                CliCommand::new(&API_METHOD_CREATE_DATASTORE)
                 .arg_param(&["name", "path"])
         )
         .insert("update",

src/bin/proxmox_file_restore/block_driver_qemu.rs

@@ -50,7 +50,7 @@ impl VMStateMap {
     /// Acquire a lock on the state map and retrieve a deserialized version
     fn load() -> Result<Self, Error> {
         let mut file = Self::open_file_raw(true)?;
-        lock_file(&mut file, true, Some(std::time::Duration::from_secs(5)))?;
+        lock_file(&mut file, true, Some(std::time::Duration::from_secs(120)))?;
         let map = serde_json::from_reader(&file).unwrap_or_default();
         Ok(Self { map, file })
     }